From 0d33822d6a1524a7c11eac976b289cd1f991579d Mon Sep 17 00:00:00 2001 From: Marc-Etienne Vargenau Date: Wed, 13 Jul 2022 11:03:50 +0200 Subject: [PATCH 001/489] Valid SPDX code for Creator: (#9) Valid SPDX code for Creator: replace "User:" by "Person:". Resolves #8 Signed-off-by: Marc-Etienne Vargenau --- src/scanoss/spdxlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index b24ef803..4ddefd61 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -182,7 +182,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'name': 'SCANOSS-SBOM', 'creationInfo': { 'created': now.strftime('%Y-%m-%dT%H:%M:%S') + now.strftime('.%f')[:4] + 'Z', - 'creators': [f'Tool: SCANOSS-PY: {__version__}', f'User: {getpass.getuser()}'] + 'creators': [f'Tool: SCANOSS-PY: {__version__}', f'Person: {getpass.getuser()}'] }, 'documentNamespace': f'https://spdx.org/spdxdocs/scanoss-py-{__version__}-{md5hex}', 'packages': [] From ae140bcc3534d51f49637edadaab7739d6b287b0 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 22 Jul 2022 14:40:07 +0100 Subject: [PATCH 002/489] Added CSV output support --- src/scanoss/cli.py | 2 +- src/scanoss/csvoutput.py | 181 +++++++++++++++++++++++++++++++++++++++ src/scanoss/scanner.py | 43 +++++++++- 3 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 src/scanoss/csvoutput.py diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index beec052d..e217684d 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -67,7 +67,7 @@ def setup_args() -> None: p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file' ) p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file' ) p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) - p_scan.add_argument('--format', '-f', type=str, choices=['plain', 'cyclonedx', 'spdxlite'], + p_scan.add_argument('--format', '-f', type=str, choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], help='Result output format (optional - default: plain)' ) p_scan.add_argument('--threads', '-T', type=int, default=10, diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py new file mode 100644 index 00000000..b6cdec33 --- /dev/null +++ b/src/scanoss/csvoutput.py @@ -0,0 +1,181 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +import os.path +import sys +import csv + +from .scanossbase import ScanossBase + + +class CsvOutput(ScanossBase): + """ + CsvOutput management class + Handle all interaction with CSV formatting + """ + + def __init__(self, debug: bool = False, output_file: str = None): + """ + Initialise the CsvOutput class + """ + super().__init__(debug) + self.output_file = output_file + self.debug = debug + + def parse(self, data: json): + """ + Parse the given input (raw/plain) JSON string and return CSV summary + :param data: json - JSON object + :return: CSV dictionary + """ + if not data: + self.print_stderr('ERROR: No JSON data provided to parse.') + return None + self.print_debug(f'Processing raw results into CSV format...') + csv_dict = [] + row_id = 1 + for f in data: + file_details = data.get(f) + # print(f'File: {f}: {file_details}') + for d in file_details: + id_details = d.get("id") + if not id_details or id_details == 'none': + continue + detected = {} + if id_details == 'dependency': + dependencies = d.get("dependencies") + if not dependencies: + self.print_stderr(f'Warning: No Dependencies found for {f}: {file_details}') + continue + for deps in dependencies: + purl = deps.get("purl") + if not purl: + self.print_stderr(f'Warning: No PURL found for {f}: {deps}') + continue + detected['purls'] = purl + for field in ['component', 'version', 'latest']: + detected[field] = deps.get(field, '') + licenses = deps.get('licenses') + dc = [] + for lic in licenses: + name = lic.get("name") + if name not in dc: # Only save the license name once + dc.append(name) + detected['licenses'] = ';'.join(dc) + + else: + purls = d.get('purl') + if not purls: + self.print_stderr(f'Warning: Purl block missing for {f}: {file_details}') + continue + pa = [] + for p in purls: + self.print_debug(f'Purl: {p}') + pa.append(p) + if not pa: + self.print_stderr(f'Warning: No PURL found for {f}: {file_details}') + continue + detected['purls'] = ';'.join(pa) + for field in ['component', 'version', 'latest']: + detected[field] = d.get(field, '') + licenses = d.get('licenses') + dc = [] + for lic in licenses: + name = lic.get("name") + if name not in dc: # Only save the license name once + dc.append(lic.get("name")) + detected['licenses'] = ';'.join(dc) + # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl + csv_dict.append({'inventory_id': row_id, 'path': f, 'usage': id_details, + 'detected_component': detected.get('component'), + 'detected_license': detected.get('licenses'), + 'detected_version': detected.get('version'), 'detected_latest': detected.get('latest'), + 'purls': detected.get('purls') + }) + row_id = row_id + 1 + return csv_dict + + def produce_from_file(self, json_file: str, output_file: str = None) -> bool: + """ + Parse plain/raw input JSON file and produce CSV output + :param json_file: + :param output_file: + :return: True if successful, False otherwise + """ + if not json_file: + self.print_stderr('ERROR: No JSON file provided to parse.') + return False + if not os.path.isfile(json_file): + self.print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') + return False + with open(json_file, 'r') as f: + success = self.produce_from_str(f.read(), output_file) + return success + + def produce_from_json(self, data: json, output_file: str = None) -> bool: + """ + Produce the CSV output from the input JSON object + :param data: JSON object + :param output_file: Output file (optional) + :return: True if successful, False otherwise + """ + csv_data = self.parse(data) + if not csv_data: + self.print_stderr('ERROR: No CSV data returned for the JSON string provided.') + return False + # Header row/column details + fields = ['inventory_id', 'path', 'detected_usage', 'detected_component', 'detected_license', 'detected_version', + 'detected_latest', 'detected_purls'] + file = sys.stdout + if not output_file and self.output_file: + output_file = self.output_file + if output_file: + file = open(output_file, 'w') + writer = csv.DictWriter(file, fieldnames=fields) + writer.writeheader() # writing headers (field names) + writer.writerows(csv_data) # writing data rows + if output_file: + file.close() + + return True + + def produce_from_str(self, json_str: str, output_file: str = None) -> bool: + """ + Produce CSV output from input JSON string + :param json_str: input JSON string + :param output_file: Output file (optional) + :return: True if successful, False otherwise + """ + if not json_str: + self.print_stderr('ERROR: No JSON string provided to parse.') + return False + try: + data = json.loads(json_str) + except Exception as e: + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + return False + return self.produce_from_json(data, output_file) +# +# End of CsvOutput Class +# diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index f82ae7c9..a99553c0 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -24,6 +24,8 @@ import json import os import sys +import datetime +import pkg_resources from progress.bar import Bar from progress.spinner import Spinner @@ -32,6 +34,7 @@ from .winnowing import Winnowing from .cyclonedx import CycloneDx from .spdxlite import SpdxLite +from .csvoutput import CsvOutput from .threadedscanning import ThreadedScanning from .scancodedeps import ScancodeDeps from .threadeddependencies import ThreadedDependencies @@ -39,6 +42,8 @@ from .scantype import ScanType from .scanossbase import ScanossBase +from . import __version__ + FILTERED_DIRS = { # Folders to skip "nbproject", "nbbuild", "nbdist", "__pycache__", "venv", "_yardoc", "eggs", "wheels", "htmlcov", "__pypackages__" @@ -103,15 +108,19 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.hidden_files_folders = hidden_files_folders self.scan_options = scan_options self._skip_snippets = True if not scan_options & ScanType.SCAN_SNIPPETS.value else False + ver_details = self.__version_details() self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, all_extensions=all_extensions ) self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, - sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout + sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, + ver_details=ver_details ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) - grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace) + grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, + ver_details=ver_details + ) self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace) self.nb_threads = nb_threads if nb_threads and nb_threads > 0: @@ -227,6 +236,24 @@ def valid_json_file(json_file: str) -> bool: return False return True + @staticmethod + def __version_details() -> str: + data = None + try: + f_name = pkg_resources.resource_filename(__name__, 'data/build_date.txt') + with open(f_name, 'r') as f: + data = f.read().rstrip() + except Exception as e: + Scanner.print_stderr(f'Warning: Problem loading build time details: {e}') + if not data or len(data) == 0: + now = datetime.datetime.now() + data = f'date: {now.strftime("%Y%m%d%H%M%S")}, utime: {int(now.timestamp())}' + + Scanner.print_stderr(f'Ver Data2: {data}') + + return f'tool: scanoss-py, version: {__version__}, {data}' + + def __log_result(self, string, outfile=None): """ Logs result to file or STDOUT @@ -473,6 +500,12 @@ def __finish_scan_threaded(self) -> bool: success = spdxlite.produce_from_json(parsed_json) else: success = spdxlite.produce_from_str(raw_output) + elif self.output_format == 'csv': + csvo = CsvOutput(self.debug, self.scan_output) + if parsed_json: + success = csvo.produce_from_json(parsed_json) + else: + success = csvo.produce_from_str(raw_output) else: self.print_stderr(f'ERROR: Unknown output format: {self.output_format}') success = False @@ -627,6 +660,9 @@ def scan_wfp_file(self, file: str = None) -> bool: elif self.output_format == 'spdxlite': spdxlite = SpdxLite(self.debug, self.scan_output) success = spdxlite.produce_from_str(raw_output) + elif self.output_format == 'csv': + csvo = CsvOutput(self.debug, self.scan_output) + csvo.produce_from_str(raw_output) else: self.print_stderr(f'ERROR: Unknown output format: {self.output_format}') success = False @@ -717,6 +753,9 @@ def scan_wfp(self, wfp: str) -> bool: elif self.output_format == 'spdxlite': spdxlite = SpdxLite(self.debug, self.scan_output) success = spdxlite.produce_from_str(raw_output) + elif self.output_format == 'csv': + csvo = CsvOutput(self.debug, self.scan_output) + csvo.produce_from_str(raw_output) else: self.print_stderr(f'ERROR: Unknown output format: {self.output_format}') success = False From c561590132b4e873817e98bcb7fbeadcb117cbe2 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 22 Jul 2022 14:40:25 +0100 Subject: [PATCH 003/489] Added document describes to SPDX --- src/scanoss/spdxlite.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 4ddefd61..95db6bbd 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -185,6 +185,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'creators': [f'Tool: SCANOSS-PY: {__version__}', f'Person: {getpass.getuser()}'] }, 'documentNamespace': f'https://spdx.org/spdxdocs/scanoss-py-{__version__}-{md5hex}', + 'documentDescribes': [], 'packages': [] } for purl in raw_data: @@ -206,9 +207,11 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: comp_ver = comp.get('version') purl_ver = f'{purl}@{comp_ver}' purl_hash = hashlib.md5(f'{purl_ver}'.encode('utf-8')).hexdigest() + purl_spdx = f'SPDXRef-{purl_hash}' + data['documentDescribes'].append(purl_spdx) data['packages'].append({ 'name': comp_name, - 'SPDXID': f'SPDXRef-{purl_hash}', + 'SPDXID': purl_spdx, 'versionInfo': comp_ver, 'downloadLocation': 'NOASSERTION', # TODO Add actual download location 'homepage': comp.get('url', ''), From 7c9f4ca5a16cf845632b75a634ea9e8e07055921 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 22 Jul 2022 14:41:33 +0100 Subject: [PATCH 004/489] added test location type printing --- tests/grpc-client-test.py | 10 +++++++--- tests/scancodedeps-test.py | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/grpc-client-test.py b/tests/grpc-client-test.py index 5b4a0419..fca2d3de 100644 --- a/tests/grpc-client-test.py +++ b/tests/grpc-client-test.py @@ -39,11 +39,13 @@ def test_grpc_dep_echo(self): Test the basic echo rpc call on the local server """ if MyTestCase.TEST_LOCAL: + server_type = "local" grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: + server_type = "remote" grpc_client = ScanossGrpc(debug=True) - echo_resp = grpc_client.deps_echo('testing dep echo') - print(f'Echo Resp: {echo_resp}') + echo_resp = grpc_client.deps_echo(f'testing dep echo ({server_type})') + print(f'Echo Resp ({server_type}): {echo_resp}') self.assertIsNotNone(echo_resp) def test_grpc_get_dependencies(self): @@ -56,11 +58,13 @@ def test_grpc_get_dependencies(self): print(f'Dependency JSON: {deps}') self.assertIsNotNone(deps) if MyTestCase.TEST_LOCAL: + server_type = "local" grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: + server_type = "remote" grpc_client = ScanossGrpc(debug=True) resp = grpc_client.get_dependencies(deps) - print(f'Resp: {resp}') + print(f'Resp ({server_type}): {resp}') self.assertIsNotNone(resp) dep_files = resp.get("files") diff --git a/tests/scancodedeps-test.py b/tests/scancodedeps-test.py index 1624aa0d..90a37c2b 100644 --- a/tests/scancodedeps-test.py +++ b/tests/scancodedeps-test.py @@ -64,14 +64,16 @@ def test_threaded_scan_dir(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: + server_type = "local" grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: + server_type = "remote" grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True)) deps = threaded_deps.responses - print(f'Dependency results: {deps}') + print(f'Dependency results ({server_type}): {deps}') self.assertIsNotNone(deps) From a6ce08ab2b9e67501cab835cf32252f50de7040f Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 22 Jul 2022 15:38:17 +0100 Subject: [PATCH 005/489] fixed header name --- src/scanoss/csvoutput.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index b6cdec33..d28b131a 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -107,11 +107,11 @@ def parse(self, data: json): dc.append(lic.get("name")) detected['licenses'] = ';'.join(dc) # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl - csv_dict.append({'inventory_id': row_id, 'path': f, 'usage': id_details, + csv_dict.append({'inventory_id': row_id, 'path': f, 'detected_usage': id_details, 'detected_component': detected.get('component'), 'detected_license': detected.get('licenses'), 'detected_version': detected.get('version'), 'detected_latest': detected.get('latest'), - 'purls': detected.get('purls') + 'detected_purls': detected.get('purls') }) row_id = row_id + 1 return csv_dict From 22fb863e174425a1cebba232330b716844a6a593 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 22 Jul 2022 20:19:33 +0100 Subject: [PATCH 006/489] added support for version date reporting --- .gitignore | 1 + CHANGELOG.md | 6 ++++++ Makefile | 14 +++++++++++--- date_time.py | 34 ++++++++++++++++++++++++++++++++++ setup.py | 2 +- src/scanoss/__init__.py | 2 +- src/scanoss/scanner.py | 7 ++++--- src/scanoss/scanossapi.py | 7 +++++-- src/scanoss/scanossgrpc.py | 15 +++++++++++---- 9 files changed, 74 insertions(+), 14 deletions(-) create mode 100755 date_time.py diff --git a/.gitignore b/.gitignore index 9607215a..27297b35 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ build/ __pycache__ venv/ .idea +src/scanoss/data/build_date.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index bd796f02..64bbc5dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.0.0] - 2022-07-22 +### Added +- Added support for CSV output (--format csv) +- Added documentDescribes to SPDXLite output + ## [0.9.0] - 2022-06-09 ### Added - Added support for dependency scanning (--dependencies) @@ -92,3 +97,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [0.7.3]: https://github.com/scanoss/scanoss.py/compare/v0.7.2...v0.7.3 [0.7.4]: https://github.com/scanoss/scanoss.py/compare/v0.7.3...v0.7.4 [0.9.0]: https://github.com/scanoss/scanoss.py/compare/v0.7.4...v0.9.0 +[1.0.0]: https://github.com/scanoss/scanoss.py/compare/v0.9.0...v1.0.0 diff --git a/Makefile b/Makefile index e05fb34b..06f0c387 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,19 @@ help: ## This help .DEFAULT_GOAL := help -clean: ## Clean all dev data +clean: date_time_clean ## Clean all dev data @echo "Removing dev and distribution data..." @rm -rf dist/* build/* venv/bin/scanoss-py src/scanoss.egg-info -dev_setup: ## Setup Python dev env for the current user +date_time_clean: ## Setup blank datetime data field + @rm -rf src/scanoss/data/build_date.txt + @echo "" > src/scanoss/data/build_date.txt + +date_time: ## Setup package build date + @rm -rf src/scanoss/data/build_date.txt + python3 date_time.py + +dev_setup: date_time_clean ## Setup Python dev env for the current user @echo "Setting up dev env for the current user..." python3 setup.py develop --user @@ -30,7 +38,7 @@ dev_uninstall: ## Uninstall Python dev setup for the current user @rm -f venv/bin/scanoss-py @rm -rf src/scanoss.egg-info -dist: clean dev_uninstall ## Prepare Python package into a distribution +dist: clean dev_uninstall date_time ## Prepare Python package into a distribution @echo "Build deployable package for distribution $(VERSION)..." python3 setup.py sdist bdist_wheel twine check dist/* diff --git a/date_time.py b/date_time.py new file mode 100755 index 00000000..99280f5a --- /dev/null +++ b/date_time.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import datetime + +""" +Store the current date/time into a data field for processing +""" +if __name__ == "__main__": + now = datetime.datetime.now() + data = f'date: {now.strftime("%Y%m%d%H%M%S")}, utime: {int(now.timestamp())}' + with open('src/scanoss/data/build_date.txt', 'w') as f: + f.write(f'{data}\n') diff --git a/setup.py b/setup.py index 6d908fd5..e8b5fe84 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def get_version(rel_path): long_description_content_type='text/markdown', install_requires=["requests", "crc32c>=2.2", "binaryornot", "progress", "grpcio<=1.42.0", "protobuf<=3.19.1"], include_package_data=True, - package_data={'': ['data/*.json']}, + package_data={'': ['data/*.json', 'data/*.txt']}, classifiers=[ "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 576ce502..bfb289e6 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '0.9.0' +__version__ = '1.0.0' diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index a99553c0..5132e109 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -238,6 +238,10 @@ def valid_json_file(json_file: str) -> bool: @staticmethod def __version_details() -> str: + """ + Extract the date this version was produced + :return: version creation date string + """ data = None try: f_name = pkg_resources.resource_filename(__name__, 'data/build_date.txt') @@ -248,9 +252,6 @@ def __version_details() -> str: if not data or len(data) == 0: now = datetime.datetime.now() data = f'date: {now.strftime("%Y%m%d%H%M%S")}, utime: {int(now.timestamp())}' - - Scanner.print_stderr(f'Ver Data2: {data}') - return f'tool: scanoss-py, version: {__version__}, {data}' diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 65fea09b..42d696af 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -45,7 +45,7 @@ class ScanossApi(ScanossBase): def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None, url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - timeout: int = 120): + timeout: int = 120, ver_details: str = None): """ Initialise the SCANOSS API :param scan_type: Scan type (default identify) @@ -60,15 +60,18 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st """ super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_SCAN_URL - self.api_key = api_key + self.api_key = api_key if api_key else SCANOSS_API_KEY self.scan_type = scan_type self.sbom_path = sbom_path self.scan_format = scan_format if scan_format else 'plain' self.flags = flags self.timeout = timeout if timeout > 5 else 120 self.headers = {} + if ver_details: + self.headers['x-scanoss-client'] = ver_details if self.api_key: self.headers['X-Session'] = self.api_key + self.headers['x-api-key'] = self.api_key self.sbom = None self.load_sbom() # Load an input SBOM if one is specified if self.trace: diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 096839ec..e231bd8e 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -37,6 +37,7 @@ # DEFAULT_URL = "localhost:50051" DEFAULT_URL = "https://scanoss.com" SCANOSS_GRPC_URL = os.environ.get("SCANOSS_GRPC_URL") if os.environ.get("SCANOSS_GRPC_URL") else DEFAULT_URL +SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' class ScanossGrpc(ScanossBase): @@ -45,7 +46,7 @@ class ScanossGrpc(ScanossBase): """ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - cert: bytes = None): + cert: bytes = None, api_key: str = None, ver_details: str = None): """ :param url: @@ -57,6 +58,12 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_GRPC_URL self.url = self.url.lower() + self.api_key = api_key if api_key else SCANOSS_API_KEY + self.metadata = [] + if self.api_key: + self.metadata.append(('x-api-key', api_key)) # Set API key if we have one + if ver_details: + self.metadata.append(('x-scanoss-client', ver_details)) secure = True if self.url.startswith('https:') else False # Is it a secure connection? if self.url.startswith('http'): u = urlparse(self.url) @@ -84,7 +91,7 @@ def deps_echo(self, message: str = 'Hello there!') -> str: """ resp: EchoResponse try: - resp = self.dependencies_stub.Echo(EchoRequest(message=message)) + resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=self.metadata) except Exception as e: self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') else: @@ -120,7 +127,7 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: return None request = ParseDict(dependencies, DependencyRequest()) # Parse the JSON/Dict into the dependency object request.depth = depth - resp = self.dependencies_stub.GetDependencies(request) + resp = self.dependencies_stub.GetDependencies(request, metadata=self.metadata) except Exception as e: self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') else: @@ -146,7 +153,7 @@ def get_dependencies_str(self, dependencies: str, depth: int = 1) -> str: # TOD purls = [purl] dep_req = DependencyRequest.Files(file="package.json", purls=purls) files = [dep_req] - resp = self.dependencies_stub.GetDependencies(DependencyRequest(files=files, depth=depth)) + resp = self.dependencies_stub.GetDependencies(DependencyRequest(files=files, depth=depth), metadata=self.metadata) except Exception as e: self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') else: From 61144e4c088c08f2dec9105c6f7785f05f7f8098 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sun, 24 Jul 2022 17:59:46 +0100 Subject: [PATCH 007/489] fixed CSV license report error --- src/scanoss/__init__.py | 2 +- src/scanoss/csvoutput.py | 31 +++++++++++++++++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index bfb289e6..b24c9e5c 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.0.0' +__version__ = '1.0.2' diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index d28b131a..be1d23ad 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -78,12 +78,15 @@ def parse(self, data: json): detected[field] = deps.get(field, '') licenses = deps.get('licenses') dc = [] - for lic in licenses: - name = lic.get("name") - if name not in dc: # Only save the license name once - dc.append(name) - detected['licenses'] = ';'.join(dc) - + if licenses: + for lic in licenses: + name = lic.get("name") + if name and name not in dc: # Only save the license name once + dc.append(name) + if not dc or len(dc) == 0: + detected['licenses'] = '' + else: + detected['licenses'] = ';'.join(dc) else: purls = d.get('purl') if not purls: @@ -93,7 +96,7 @@ def parse(self, data: json): for p in purls: self.print_debug(f'Purl: {p}') pa.append(p) - if not pa: + if not pa or len(pa) == 0: self.print_stderr(f'Warning: No PURL found for {f}: {file_details}') continue detected['purls'] = ';'.join(pa) @@ -101,11 +104,15 @@ def parse(self, data: json): detected[field] = d.get(field, '') licenses = d.get('licenses') dc = [] - for lic in licenses: - name = lic.get("name") - if name not in dc: # Only save the license name once - dc.append(lic.get("name")) - detected['licenses'] = ';'.join(dc) + if licenses: + for lic in licenses: + name = lic.get("name") + if name and name not in dc: # Only save the license name once + dc.append(lic.get("name")) + if not dc or len(dc) == 0: + detected['licenses'] = '' + else: + detected['licenses'] = ';'.join(dc) # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl csv_dict.append({'inventory_id': row_id, 'path': f, 'detected_usage': id_details, 'detected_component': detected.get('component'), From e6a2fb3895fd4d6e7ccb9dc78d5625fcb152e33b Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 27 Jul 2022 23:05:35 +0100 Subject: [PATCH 008/489] Fixing broken symlink checks when fingerprinting --- src/scanoss/__init__.py | 2 +- src/scanoss/scanner.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b24c9e5c..f30ba79e 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.0.2' +__version__ = '1.0.3' diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 5132e109..9458f4a5 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -365,8 +365,8 @@ def scan_folder(self, scan_dir: str) -> bool: f_size = 0 try: f_size = os.stat(path).st_size - except: - self.print_trace(f'Ignoring missing symlink file: {file}') # Can fail if there is a broken symlink + except Exception as e: + self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink if f_size > 0: # Ignore broken links and empty files self.print_trace(f'Fingerprinting {path}...') if spinner: @@ -801,8 +801,12 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): self.print_trace(f'Root: {root}, Dirs: {dirs}, Files {filtered_files}') for file in filtered_files: path = os.path.join(root, file) - file_stat = os.stat(path) - if file_stat.st_size > 0: # Ignore empty files + f_size = 0 + try: + f_size = os.stat(path).st_size + except Exception as e: + self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink + if f_size > 0: # Ignore empty files self.print_debug(f'Fingerprinting {path}...') wfps += self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) if wfps: From ff3a8454817791860d2206e348ff2360849bc073 Mon Sep 17 00:00:00 2001 From: Alejandro Perez Date: Wed, 31 Aug 2022 20:19:11 -0300 Subject: [PATCH 009/489] Fixed category name in referenceCategory field of SPDXlite. --- src/scanoss/spdxlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 95db6bbd..75e2a524 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -220,7 +220,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'filesAnalyzed': False, 'copyrightText': 'NOASSERTION', 'externalRefs': [{ - 'referenceCategory': 'PACKAGE_MANAGER', + 'referenceCategory': 'PACKAGE-MANAGER', 'referenceLocator': purl_ver, 'referenceType': 'purl' }] From 917cd658fa1d66e259970db44c6943e8a2f4221d Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 7 Sep 2022 09:45:24 +0100 Subject: [PATCH 010/489] adjusted protobuf dependency requirements --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4d5ca4f4..db3b752f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ crc32c>=2.2 binaryornot progress grpcio<=1.42.0 -protobuf<=3.19.1 +protobuf>=3.16.0,<=3.19.1 diff --git a/setup.py b/setup.py index e8b5fe84..7eee6281 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ def get_version(rel_path): description='Simple Python library to use the SCANOSS APIs.', long_description=read("PACKAGE.md"), long_description_content_type='text/markdown', - install_requires=["requests", "crc32c>=2.2", "binaryornot", "progress", "grpcio<=1.42.0", "protobuf<=3.19.1"], + install_requires=["requests", "crc32c>=2.2", "binaryornot", "progress", "grpcio<=1.42.0", "protobuf>=3.16.0,<=3.19.1"], include_package_data=True, package_data={'': ['data/*.json', 'data/*.txt']}, classifiers=[ From 04c32dcdd1f317ab1d6aa3569ef043eafb92ae97 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 7 Sep 2022 09:45:57 +0100 Subject: [PATCH 011/489] added timeouts to gRPC comms --- src/scanoss/scanossapi.py | 2 +- src/scanoss/scanossgrpc.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 42d696af..4be77a9e 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -62,8 +62,8 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st self.url = url if url else SCANOSS_SCAN_URL self.api_key = api_key if api_key else SCANOSS_API_KEY self.scan_type = scan_type - self.sbom_path = sbom_path self.scan_format = scan_format if scan_format else 'plain' + self.sbom_path = sbom_path self.flags = flags self.timeout = timeout if timeout > 5 else 120 self.headers = {} diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index e231bd8e..0dbd094b 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -91,7 +91,7 @@ def deps_echo(self, message: str = 'Hello there!') -> str: """ resp: EchoResponse try: - resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=self.metadata) + resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=self.metadata, timeout=3) except Exception as e: self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') else: @@ -127,7 +127,7 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: return None request = ParseDict(dependencies, DependencyRequest()) # Parse the JSON/Dict into the dependency object request.depth = depth - resp = self.dependencies_stub.GetDependencies(request, metadata=self.metadata) + resp = self.dependencies_stub.GetDependencies(request, metadata=self.metadata, timeout=600) except Exception as e: self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') else: @@ -153,7 +153,7 @@ def get_dependencies_str(self, dependencies: str, depth: int = 1) -> str: # TOD purls = [purl] dep_req = DependencyRequest.Files(file="package.json", purls=purls) files = [dep_req] - resp = self.dependencies_stub.GetDependencies(DependencyRequest(files=files, depth=depth), metadata=self.metadata) + resp = self.dependencies_stub.GetDependencies(DependencyRequest(files=files, depth=depth), metadata=self.metadata, timeout=600) except Exception as e: self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') else: From bedb980fe4bd153f88c188be86ba0afd3bab28dd Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 7 Sep 2022 09:46:41 +0100 Subject: [PATCH 012/489] fixed SPDX spelling mistake, updated protobuf version reqs, added grpc timeouts --- CHANGELOG.md | 6 ++++++ src/scanoss/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64bbc5dc..f1e977d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.0.4] - 2022-09-07 +### Fixed +- Fixed spelling mistake in SPDX output +- Adjusted protobuf module requirements + ## [1.0.0] - 2022-07-22 ### Added - Added support for CSV output (--format csv) @@ -98,3 +103,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [0.7.4]: https://github.com/scanoss/scanoss.py/compare/v0.7.3...v0.7.4 [0.9.0]: https://github.com/scanoss/scanoss.py/compare/v0.7.4...v0.9.0 [1.0.0]: https://github.com/scanoss/scanoss.py/compare/v0.9.0...v1.0.0 +[1.0.4]: https://github.com/scanoss/scanoss.py/compare/v1.0.0...v1.0.4 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index f30ba79e..49fe8ee4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.0.3' +__version__ = '1.0.4' From 659f0b670acfcf2546e0a29982b3af16682d7e31 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 16 Sep 2022 08:35:33 +0100 Subject: [PATCH 013/489] initialise scancode during image creation. --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index b11d7812..51463163 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,5 +36,8 @@ ENV GRPC_POLL_STRATEGY=poll VOLUME /scanoss WORKDIR /scanoss +# Run scancode once to setup any initial files, etc. so that it'll run faster later +RUN scancode -p --only-findings --quiet --json /scanoss/scancode-dependencies.json /scanoss && rm -f /scanoss/scancode-dependencies.json + ENTRYPOINT ["scanoss-py"] CMD ["--help"] From 6ced7de37fcc6ef48aced28be6e5a88ff57d7a70 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 6 Oct 2022 15:13:29 +0100 Subject: [PATCH 014/489] added support for scancode output format 2.0 --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- src/scanoss/scancodedeps.py | 12 ++++++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e977d8..4b364fb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.0.6] - 2022-09-19 +### Added +- Added support for scancode 2.0 output format + ## [1.0.4] - 2022-09-07 ### Fixed - Fixed spelling mistake in SPDX output @@ -104,3 +108,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [0.9.0]: https://github.com/scanoss/scanoss.py/compare/v0.7.4...v0.9.0 [1.0.0]: https://github.com/scanoss/scanoss.py/compare/v0.9.0...v1.0.0 [1.0.4]: https://github.com/scanoss/scanoss.py/compare/v1.0.0...v1.0.4 +[1.0.5]: https://github.com/scanoss/scanoss.py/compare/v1.0.4...v1.0.6 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 49fe8ee4..065cc485 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.0.4' +__version__ = '1.0.6' diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index db08ee02..2a4ca0c2 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -98,9 +98,11 @@ def produce_from_json(self, data: json) -> dict: f_type = fd.get('type') if not f_type or f_type == '' or f_type != 'file': # Only process files continue - f_packages = fd.get('packages') + f_packages = fd.get('package_data') # scancode format 2.0 if not f_packages or f_packages == '': - continue + f_packages = fd.get('packages') # scancode formate 1.0 + if not f_packages or f_packages == '': + continue # print(f'Path: {f_path}, Packages: {f_packages}') for pkgs in f_packages: pk_deps = pkgs.get('dependencies') @@ -113,7 +115,9 @@ def produce_from_json(self, data: json) -> dict: if not dp or dp == '': continue dp_data = {'purl': dp} - rq = d.get('requirement') + rq = d.get('extracted_requirement') # scancode format 2.0 + if not rq or rq == '': + rq = d.get('requirement') # scancode format 1.0 if rq and rq != '' and not dp.endswith(rq): dp_data['requirement'] = rq purls.append(dp_data) @@ -193,7 +197,7 @@ def run_scan(self, output_file: str = None, what_to_scan: str = None) -> bool: open(output_file, 'w').close() self.print_trace(f'About to execute {self.sc_command} -p --only-findings --quiet --json {output_file}' f' {what_to_scan}') - result = subprocess.run([self.sc_command, '-p', '--only-findings', '--quiet', '--json', + result = subprocess.run([self.sc_command, '-p', '--only-findings', '--quiet', '--strip-root', '--json', output_file, what_to_scan], cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=self.timeout From f56480ab56d28e15398e6455c03f4dfa124ad51c Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 6 Oct 2022 15:14:27 +0100 Subject: [PATCH 015/489] added extra API debug details --- src/scanoss/scanner.py | 4 ++-- src/scanoss/scanossapi.py | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 9458f4a5..a81ab8aa 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -610,7 +610,7 @@ def scan_wfp_file(self, file: str = None) -> bool: if l_size >= self.max_post_size and wfp: self.print_debug(f'Sending {batch_files} ({cur_files}) of' f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.') - if cur_size > self.max_post_size: + if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store if bar: @@ -703,7 +703,7 @@ def scan_wfp_file_threaded(self, file: str = None) -> bool: l_size = cur_size + len(scan_block.encode('utf-8')) # Hit the max post size, so sending the current batch and continue processing if l_size >= self.max_post_size and wfp: - if cur_size > self.max_post_size: + if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') self.threaded_scan.queue_add(wfp) queue_size += 1 diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 4be77a9e..35f5156f 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -118,13 +118,13 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): timeout=self.timeout) except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: if retry > 5: # Timed out 5 or more times, fail - self.print_stderr(f'ERROR: Timeout/Connection Error POSTing data: {scan_files}') - raise Exception(f"ERROR: The SCANOSS API request timed out for {self.url}") from e + self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data: {scan_files}') + raise Exception(f"ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for {self.url}") from e else: - self.print_stderr(f'Warning: Timeout/Connection Error communicating with {self.url}. Retrying...') + self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...') time.sleep(5) except Exception as e: - self.print_stderr(f'ERROR: Exception POSTing data: {scan_files}') + self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data: {scan_files}') raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e else: if not r: @@ -151,7 +151,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): json_resp = r.json() return json_resp except (JSONDecodeError, Exception) as e: - self.print_stderr(f'ERROR: The SCANOSS API returned an invalid JSON: {e}') + self.print_stderr(f'ERROR: The SCANOSS API returned an invalid JSON ({e.__class__.__name__}): {e}') ctime = int(time.time()) bad_json_file = f'bad_json-{scan_id}-{ctime}.txt' if scan_id else f'bad_json-{ctime}.txt' self.print_stderr(f'Ignoring result. Please look in "{bad_json_file}" for more details.') @@ -161,7 +161,8 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): f.write(r.text) f.write("---Bad JSON End---\n") except Exception as ee: - self.print_stderr(f'Warning: Issue writing bad json file - {bad_json_file}: {ee}') + self.print_stderr(f'Warning: Issue writing bad json file - {bad_json_file} ({ee.__class__.__name__}):' + f' {ee}') return None # From e78f14718310e9120f60d83d02dc828048579397 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 12 Oct 2022 09:39:10 +0100 Subject: [PATCH 016/489] Update CDX output to support version 1.4 and non-SPDX licenses --- src/scanoss/cyclonedx.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 81be87c0..7f4a420d 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -24,10 +24,10 @@ import json import os.path import sys -import hashlib -import time +import uuid from .scanossbase import ScanossBase +from .spdxlite import SpdxLite class CycloneDx(ScanossBase): @@ -40,9 +40,9 @@ def __init__(self, debug: bool = False, output_file: str = None): """ Initialise the CycloneDX class """ - super().__init__(debug) self.output_file = output_file self.debug = debug + self._spdx = SpdxLite(debug=debug) def parse(self, data: json): """ @@ -143,12 +143,17 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: if not cdx: self.print_stderr('ERROR: No CycloneDX data returned for the JSON string provided.') return False - md5hex = hashlib.md5(f'{time.time()}'.encode('utf-8')).hexdigest() + self._spdx.load_license_data() # Load SPDX license name data for later reference + # + # Using CDX version 1.4: https://cyclonedx.org/docs/1.4/json/ + # Validate using: https://github.com/CycloneDX/cyclonedx-cli + # cyclonedx-cli validate --input-format json --input-version v1_4 --fail-on-errors --input-file cdx.json + # data = { 'bomFormat': 'CycloneDX', - 'specVersion': '1.2', - 'serialNumber': f'scanoss:SCANOSS-PY - SCANOSS CLI-{md5hex}', - 'version': '1', + 'specVersion': '1.4', + 'serialNumber': f'urn:uuid:{uuid.uuid4()}', + 'version': 1, 'components': [] } for purl in cdx: @@ -156,9 +161,18 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: lic_text = [] licenses = comp.get('licenses') if licenses: - for lic in licenses: - lic_text.append({'license': {'id': lic.get('id')}}) - m_type = 'Snippet' if comp.get('id') == 'snippet' else 'Library' + lic_set = set() + for lic in licenses: # Get a unique set of licenses + lc_id = lic.get('id') + spdx_id = self._spdx.get_spdx_license_id(lc_id) + lic_set.add(spdx_id if spdx_id else lc_id) + for lc_id in lic_set: # Store licenses for later inclusion + spdx_id = self._spdx.get_spdx_license_id(lc_id) + if not spdx_id: + lic_text.append({'license': {'name': lc_id}}) # Not an SPDX license, so store it by name + else: + lic_text.append({'license': {'id': spdx_id}}) + m_type = 'library' data['components'].append({ 'type': m_type, 'name': comp.get('component'), @@ -166,11 +180,6 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'version': comp.get('version'), 'purl': purl, 'licenses': lic_text - # 'licenses': [{ - # 'license': { - # 'id': comp.get('license') - # } - # }] }) # End for loop file = sys.stdout From b6f0ca5bb1d8c43211894829fcda08c2fb75d4f8 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 12 Oct 2022 09:39:51 +0100 Subject: [PATCH 017/489] Added request ID to gRPC requests --- src/scanoss/scanossgrpc.py | 72 ++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 0dbd094b..7c1e4f96 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -23,6 +23,8 @@ """ import os +import uuid + import grpc import json from urllib.parse import urlparse @@ -89,15 +91,31 @@ def deps_echo(self, message: str = 'Hello there!') -> str: :param message: Message to send (default: Hello there!) :return: echo or None """ + request_id = str(uuid.uuid4()) resp: EchoResponse try: - resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=self.metadata, timeout=3) + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + # resp, call = self.dependencies_stub.Echo.with_call(EchoRequest(message=message), metadata=metadata, timeout=3) + resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3) except Exception as e: - self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') else: + # self.print_stderr(f'resp: {resp} - call: {call}') + # response_id = "" + # if not call: + # self.print_stderr(f'No call to leverage.') + # for key, value in call.trailing_metadata(): + # print('Greeter client received trailing metadata: key=%s value=%s' % (key, value)) + # + # for key, value in call.trailing_metadata(): + # if key == 'x-response-id': + # response_id = value + # self.print_stderr(f'Response ID: {response_id}. Metadata: {call.trailing_metadata()}') if resp: return resp.message - self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}') + self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}') return None def get_dependencies(self, dependencies: json, depth: int = 1) -> dict: @@ -119,6 +137,7 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: if not dependencies: self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') return None + request_id = str(uuid.uuid4()) resp: DependencyResponse try: files_json = dependencies.get("files") @@ -127,56 +146,33 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: return None request = ParseDict(dependencies, DependencyRequest()) # Parse the JSON/Dict into the dependency object request.depth = depth - resp = self.dependencies_stub.GetDependencies(request, metadata=self.metadata, timeout=600) - except Exception as e: - self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') - else: - if resp: - if not self._check_status_response(resp.status): - return None - return MessageToDict(resp, preserving_proto_field_name=True) # Convert the gRPC response to a dictionary - return None - - def get_dependencies_str(self, dependencies: str, depth: int = 1) -> str: # TODO remove? - """ - Client function to call the rpc for GetDependencies - :param dependencies: Message to send to the service - :param depth: depth of sub-dependencies to search (default: 1) - :return: Server response or None - """ - if not dependencies: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') - return None - resp: DependencyResponse - try: - purl = DependencyRequest.Purls(purl="pkg", requirement="^1.0") - purls = [purl] - dep_req = DependencyRequest.Files(file="package.json", purls=purls) - files = [dep_req] - resp = self.dependencies_stub.GetDependencies(DependencyRequest(files=files, depth=depth), metadata=self.metadata, timeout=600) + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...') + resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=600) except Exception as e: - self.print_stderr(f'ERROR: Problem encountered sending gRPC message: {e}') + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') else: if resp: - if not self._check_status_response(resp.status): + if not self._check_status_response(resp.status, request_id): return None - return resp.dependencies + return MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dictionary return None - def _check_status_response(self, status_response: StatusResponse) -> bool: + def _check_status_response(self, status_response: StatusResponse, request_id: str = None) -> bool: """ Check the response object to see if the command was successful or not :param status_response: Status Response :return: True if successful, False otherwise """ if not status_response: - self.print_stderr('Warning: No status response supplied. Assuming it was ok.') + self.print_stderr(f'Warning: No status response supplied (rqId: {request_id}). Assuming it was ok.') return True - self.print_debug(f'Checking response status: {status_response}') + self.print_debug(f'Checking response status (rqId: {request_id}): {status_response}') status_code: StatusCode = status_response.status - # self.print_stderr(f'Status Code: {status_code}, Message: {status_response.message}') if status_code > 1: - self.print_stderr(f'Not such a success: {status_response.message}') + self.print_stderr(f'Not such a success (rqId: {request_id}): {status_response.message}') return False return True # From 518482fc85a85c630a6dc05ee120c8033897b1a1 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 12 Oct 2022 09:40:22 +0100 Subject: [PATCH 018/489] added latest SPDX license definitions --- src/scanoss/data/spdx-exceptions.json | 470 +- src/scanoss/data/spdx-licenses.json | 6234 +++++++++++++------------ 2 files changed, 3484 insertions(+), 3220 deletions(-) diff --git a/src/scanoss/data/spdx-exceptions.json b/src/scanoss/data/spdx-exceptions.json index f9cef8f5..2c7e07fe 100644 --- a/src/scanoss/data/spdx-exceptions.json +++ b/src/scanoss/data/spdx-exceptions.json @@ -1,466 +1,500 @@ { - "licenseListVersion": "f9911cd", + "licenseListVersion": "03c58ca", "exceptions": [ { - "reference": "./OCCT-exception-1.0.json", + "reference": "./Fawkes-Runtime-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./OCCT-exception-1.0.html", + "detailsUrl": "./Fawkes-Runtime-exception.html", "referenceNumber": 1, - "name": "Open CASCADE Exception 1.0", - "licenseExceptionId": "OCCT-exception-1.0", + "name": "Fawkes Runtime Exception", + "licenseExceptionId": "Fawkes-Runtime-exception", "seeAlso": [ - "http://www.opencascade.com/content/licensing" + "http://www.fawkesrobotics.org/about/license/" ] }, { - "reference": "./Libtool-exception.json", + "reference": "./Autoconf-exception-3.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Libtool-exception.html", + "detailsUrl": "./Autoconf-exception-3.0.html", "referenceNumber": 2, - "name": "Libtool Exception", - "licenseExceptionId": "Libtool-exception", + "name": "Autoconf exception 3.0", + "licenseExceptionId": "Autoconf-exception-3.0", "seeAlso": [ - "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4" + "http://www.gnu.org/licenses/autoconf-exception-3.0.html" ] }, { - "reference": "./gnu-javamail-exception.json", + "reference": "./FLTK-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./gnu-javamail-exception.html", + "detailsUrl": "./FLTK-exception.html", "referenceNumber": 3, - "name": "GNU JavaMail exception", - "licenseExceptionId": "gnu-javamail-exception", + "name": "FLTK exception", + "licenseExceptionId": "FLTK-exception", "seeAlso": [ - "http://www.gnu.org/software/classpathx/javamail/javamail.html" + "http://www.fltk.org/COPYING.php" ] }, { - "reference": "./SHL-2.1.json", + "reference": "./u-boot-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./SHL-2.1.html", + "detailsUrl": "./u-boot-exception-2.0.html", "referenceNumber": 4, - "name": "Solderpad Hardware License v2.1", - "licenseExceptionId": "SHL-2.1", + "name": "U-Boot exception 2.0", + "licenseExceptionId": "u-boot-exception-2.0", "seeAlso": [ - "https://solderpad.org/licenses/SHL-2.1/" + "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003dLicenses/Exceptions" ] }, { - "reference": "./Linux-syscall-note.json", + "reference": "./CLISP-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Linux-syscall-note.html", + "detailsUrl": "./CLISP-exception-2.0.html", "referenceNumber": 5, - "name": "Linux Syscall Note", - "licenseExceptionId": "Linux-syscall-note", + "name": "CLISP exception 2.0", + "licenseExceptionId": "CLISP-exception-2.0", "seeAlso": [ - "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/COPYING" + "http://sourceforge.net/p/clisp/clisp/ci/default/tree/COPYRIGHT" ] }, { - "reference": "./Bootloader-exception.json", + "reference": "./WxWindows-exception-3.1.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Bootloader-exception.html", + "detailsUrl": "./WxWindows-exception-3.1.html", "referenceNumber": 6, - "name": "Bootloader Distribution Exception", - "licenseExceptionId": "Bootloader-exception", + "name": "WxWindows Library Exception 3.1", + "licenseExceptionId": "WxWindows-exception-3.1", "seeAlso": [ - "https://github.com/pyinstaller/pyinstaller/blob/develop/COPYING.txt" + "http://www.opensource.org/licenses/WXwindows" ] }, { - "reference": "./CLISP-exception-2.0.json", + "reference": "./389-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./CLISP-exception-2.0.html", + "detailsUrl": "./389-exception.html", "referenceNumber": 7, - "name": "CLISP exception 2.0", - "licenseExceptionId": "CLISP-exception-2.0", + "name": "389 Directory Server Exception", + "licenseExceptionId": "389-exception", "seeAlso": [ - "http://sourceforge.net/p/clisp/clisp/ci/default/tree/COPYRIGHT" + "http://directory.fedoraproject.org/wiki/GPL_Exception_License_Text", + "https://web.archive.org/web/20080828121337/http://directory.fedoraproject.org/wiki/GPL_Exception_License_Text" ] }, { - "reference": "./GCC-exception-3.1.json", + "reference": "./SHL-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GCC-exception-3.1.html", + "detailsUrl": "./SHL-2.0.html", "referenceNumber": 8, - "name": "GCC Runtime Library exception 3.1", - "licenseExceptionId": "GCC-exception-3.1", + "name": "Solderpad Hardware License v2.0", + "licenseExceptionId": "SHL-2.0", "seeAlso": [ - "http://www.gnu.org/licenses/gcc-exception-3.1.html" + "https://solderpad.org/licenses/SHL-2.0/" ] }, { - "reference": "./OpenJDK-assembly-exception-1.0.json", + "reference": "./DigiRule-FOSS-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./OpenJDK-assembly-exception-1.0.html", + "detailsUrl": "./DigiRule-FOSS-exception.html", "referenceNumber": 9, - "name": "OpenJDK Assembly exception 1.0", - "licenseExceptionId": "OpenJDK-assembly-exception-1.0", + "name": "DigiRule FOSS License Exception", + "licenseExceptionId": "DigiRule-FOSS-exception", "seeAlso": [ - "http://openjdk.java.net/legal/assembly-exception.html" + "http://www.digirulesolutions.com/drupal/foss" ] }, { - "reference": "./WxWindows-exception-3.1.json", + "reference": "./KiCad-libraries-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./WxWindows-exception-3.1.html", + "detailsUrl": "./KiCad-libraries-exception.html", "referenceNumber": 10, - "name": "WxWindows Library Exception 3.1", - "licenseExceptionId": "WxWindows-exception-3.1", + "name": "KiCad Libraries Exception", + "licenseExceptionId": "KiCad-libraries-exception", "seeAlso": [ - "http://www.opensource.org/licenses/WXwindows" + "https://www.kicad.org/libraries/license/" ] }, { - "reference": "./eCos-exception-2.0.json", + "reference": "./GStreamer-exception-2005.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./eCos-exception-2.0.html", + "detailsUrl": "./GStreamer-exception-2005.html", "referenceNumber": 11, - "name": "eCos exception 2.0", - "licenseExceptionId": "eCos-exception-2.0", + "name": "GStreamer Exception (2005)", + "licenseExceptionId": "GStreamer-exception-2005", "seeAlso": [ - "http://ecos.sourceware.org/license-overview.html" + "https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language\u003dc#licensing-of-applications-using-gstreamer" ] }, { - "reference": "./LLVM-exception.json", + "reference": "./Qwt-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./LLVM-exception.html", + "detailsUrl": "./Qwt-exception-1.0.html", "referenceNumber": 12, - "name": "LLVM Exception", - "licenseExceptionId": "LLVM-exception", + "name": "Qwt exception 1.0", + "licenseExceptionId": "Qwt-exception-1.0", "seeAlso": [ - "http://llvm.org/foundation/relicensing/LICENSE.txt" + "http://qwt.sourceforge.net/qwtlicense.html" ] }, { - "reference": "./GPL-CC-1.0.json", + "reference": "./GPL-3.0-linking-source-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GPL-CC-1.0.html", + "detailsUrl": "./GPL-3.0-linking-source-exception.html", "referenceNumber": 13, - "name": "GPL Cooperation Commitment 1.0", - "licenseExceptionId": "GPL-CC-1.0", + "name": "GPL-3.0 Linking Exception (with Corresponding Source)", + "licenseExceptionId": "GPL-3.0-linking-source-exception", "seeAlso": [ - "https://github.com/gplcc/gplcc/blob/master/Project/COMMITMENT", - "https://gplcc.github.io/gplcc/Project/README-PROJECT.html" + "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs", + "https://github.com/mirror/wget/blob/master/src/http.c#L20" ] }, { - "reference": "./mif-exception.json", + "reference": "./OCaml-LGPL-linking-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./mif-exception.html", + "detailsUrl": "./OCaml-LGPL-linking-exception.html", "referenceNumber": 14, - "name": "Macros and Inline Functions Exception", - "licenseExceptionId": "mif-exception", + "name": "OCaml LGPL Linking Exception", + "licenseExceptionId": "OCaml-LGPL-linking-exception", "seeAlso": [ - "http://www.scs.stanford.edu/histar/src/lib/cppsup/exception", - "http://dev.bertos.org/doxygen/", - "https://www.threadingbuildingblocks.org/licensing" + "https://caml.inria.fr/ocaml/license.en.html" ] }, { - "reference": "./Classpath-exception-2.0.json", + "reference": "./gnu-javamail-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Classpath-exception-2.0.html", + "detailsUrl": "./gnu-javamail-exception.html", "referenceNumber": 15, - "name": "Classpath exception 2.0", - "licenseExceptionId": "Classpath-exception-2.0", + "name": "GNU JavaMail exception", + "licenseExceptionId": "gnu-javamail-exception", "seeAlso": [ - "http://www.gnu.org/software/classpath/license.html", - "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception" + "http://www.gnu.org/software/classpathx/javamail/javamail.html" ] }, { - "reference": "./SHL-2.0.json", + "reference": "./Libtool-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./SHL-2.0.html", + "detailsUrl": "./Libtool-exception.html", "referenceNumber": 16, - "name": "Solderpad Hardware License v2.0", - "licenseExceptionId": "SHL-2.0", + "name": "Libtool Exception", + "licenseExceptionId": "Libtool-exception", "seeAlso": [ - "https://solderpad.org/licenses/SHL-2.0/" + "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4" ] }, { - "reference": "./Qwt-exception-1.0.json", + "reference": "./LGPL-3.0-linking-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Qwt-exception-1.0.html", + "detailsUrl": "./LGPL-3.0-linking-exception.html", "referenceNumber": 17, - "name": "Qwt exception 1.0", - "licenseExceptionId": "Qwt-exception-1.0", + "name": "LGPL-3.0 Linking Exception", + "licenseExceptionId": "LGPL-3.0-linking-exception", "seeAlso": [ - "http://qwt.sourceforge.net/qwtlicense.html" + "https://raw.githubusercontent.com/go-xmlpath/xmlpath/v2/LICENSE", + "https://github.com/goamz/goamz/blob/master/LICENSE", + "https://github.com/juju/errors/blob/master/LICENSE" ] }, { - "reference": "./i2p-gpl-java-exception.json", + "reference": "./mif-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./i2p-gpl-java-exception.html", + "detailsUrl": "./mif-exception.html", "referenceNumber": 18, - "name": "i2p GPL+Java Exception", - "licenseExceptionId": "i2p-gpl-java-exception", + "name": "Macros and Inline Functions Exception", + "licenseExceptionId": "mif-exception", "seeAlso": [ - "http://geti2p.net/en/get-involved/develop/licenses#java_exception" + "http://www.scs.stanford.edu/histar/src/lib/cppsup/exception", + "http://dev.bertos.org/doxygen/", + "https://www.threadingbuildingblocks.org/licensing" ] }, { - "reference": "./Autoconf-exception-3.0.json", + "reference": "./openvpn-openssl-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Autoconf-exception-3.0.html", + "detailsUrl": "./openvpn-openssl-exception.html", "referenceNumber": 19, - "name": "Autoconf exception 3.0", - "licenseExceptionId": "Autoconf-exception-3.0", + "name": "OpenVPN OpenSSL Exception", + "licenseExceptionId": "openvpn-openssl-exception", "seeAlso": [ - "http://www.gnu.org/licenses/autoconf-exception-3.0.html" + "http://openvpn.net/index.php/license.html" ] }, { - "reference": "./OCaml-LGPL-linking-exception.json", + "reference": "./GCC-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./OCaml-LGPL-linking-exception.html", + "detailsUrl": "./GCC-exception-2.0.html", "referenceNumber": 20, - "name": "OCaml LGPL Linking Exception", - "licenseExceptionId": "OCaml-LGPL-linking-exception", + "name": "GCC Runtime Library exception 2.0", + "licenseExceptionId": "GCC-exception-2.0", "seeAlso": [ - "https://caml.inria.fr/ocaml/license.en.html" + "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" ] }, { - "reference": "./FLTK-exception.json", + "reference": "./Bison-exception-2.2.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./FLTK-exception.html", + "detailsUrl": "./Bison-exception-2.2.html", "referenceNumber": 21, - "name": "FLTK exception", - "licenseExceptionId": "FLTK-exception", + "name": "Bison exception 2.2", + "licenseExceptionId": "Bison-exception-2.2", "seeAlso": [ - "http://www.fltk.org/COPYING.php" + "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" ] }, { - "reference": "./GPL-3.0-linking-exception.json", + "reference": "./eCos-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GPL-3.0-linking-exception.html", + "detailsUrl": "./eCos-exception-2.0.html", "referenceNumber": 22, - "name": "GPL-3.0 Linking Exception", - "licenseExceptionId": "GPL-3.0-linking-exception", + "name": "eCos exception 2.0", + "licenseExceptionId": "eCos-exception-2.0", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs" + "http://ecos.sourceware.org/license-overview.html" ] }, { - "reference": "./Universal-FOSS-exception-1.0.json", + "reference": "./LLVM-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Universal-FOSS-exception-1.0.html", + "detailsUrl": "./LLVM-exception.html", "referenceNumber": 23, - "name": "Universal FOSS Exception, Version 1.0", - "licenseExceptionId": "Universal-FOSS-exception-1.0", + "name": "LLVM Exception", + "licenseExceptionId": "LLVM-exception", "seeAlso": [ - "https://oss.oracle.com/licenses/universal-foss-exception/" + "http://llvm.org/foundation/relicensing/LICENSE.txt" ] }, { - "reference": "./LZMA-exception.json", + "reference": "./i2p-gpl-java-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./LZMA-exception.html", + "detailsUrl": "./i2p-gpl-java-exception.html", "referenceNumber": 24, - "name": "LZMA exception", - "licenseExceptionId": "LZMA-exception", + "name": "i2p GPL+Java Exception", + "licenseExceptionId": "i2p-gpl-java-exception", "seeAlso": [ - "http://nsis.sourceforge.net/Docs/AppendixI.html#I.6" + "http://geti2p.net/en/get-involved/develop/licenses#java_exception" ] }, { - "reference": "./DigiRule-FOSS-exception.json", + "reference": "./GPL-CC-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./DigiRule-FOSS-exception.html", + "detailsUrl": "./GPL-CC-1.0.html", "referenceNumber": 25, - "name": "DigiRule FOSS License Exception", - "licenseExceptionId": "DigiRule-FOSS-exception", + "name": "GPL Cooperation Commitment 1.0", + "licenseExceptionId": "GPL-CC-1.0", "seeAlso": [ - "http://www.digirulesolutions.com/drupal/foss" + "https://github.com/gplcc/gplcc/blob/master/Project/COMMITMENT", + "https://gplcc.github.io/gplcc/Project/README-PROJECT.html" ] }, { - "reference": "./GPL-3.0-linking-source-exception.json", + "reference": "./Classpath-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GPL-3.0-linking-source-exception.html", + "detailsUrl": "./Classpath-exception-2.0.html", "referenceNumber": 26, - "name": "GPL-3.0 Linking Exception (with Corresponding Source)", - "licenseExceptionId": "GPL-3.0-linking-source-exception", + "name": "Classpath exception 2.0", + "licenseExceptionId": "Classpath-exception-2.0", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs", - "https://github.com/mirror/wget/blob/master/src/http.c#L20" + "http://www.gnu.org/software/classpath/license.html", + "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception" ] }, { - "reference": "./u-boot-exception-2.0.json", + "reference": "./freertos-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./u-boot-exception-2.0.html", + "detailsUrl": "./freertos-exception-2.0.html", "referenceNumber": 27, - "name": "U-Boot exception 2.0", - "licenseExceptionId": "u-boot-exception-2.0", + "name": "FreeRTOS Exception 2.0", + "licenseExceptionId": "freertos-exception-2.0", "seeAlso": [ - "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003dLicenses/Exceptions" + "https://web.archive.org/web/20060809182744/http://www.freertos.org/a00114.html" ] }, { - "reference": "./GCC-exception-2.0.json", + "reference": "./Qt-GPL-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GCC-exception-2.0.html", + "detailsUrl": "./Qt-GPL-exception-1.0.html", "referenceNumber": 28, - "name": "GCC Runtime Library exception 2.0", - "licenseExceptionId": "GCC-exception-2.0", + "name": "Qt GPL exception 1.0", + "licenseExceptionId": "Qt-GPL-exception-1.0", "seeAlso": [ - "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" + "http://code.qt.io/cgit/qt/qtbase.git/tree/LICENSE.GPL3-EXCEPT" ] }, { - "reference": "./PS-or-PDF-font-exception-20170817.json", + "reference": "./GStreamer-exception-2008.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./PS-or-PDF-font-exception-20170817.html", + "detailsUrl": "./GStreamer-exception-2008.html", "referenceNumber": 29, - "name": "PS/PDF font exception (2017-08-17)", - "licenseExceptionId": "PS-or-PDF-font-exception-20170817", + "name": "GStreamer Exception (2008)", + "licenseExceptionId": "GStreamer-exception-2008", "seeAlso": [ - "https://github.com/ArtifexSoftware/urw-base35-fonts/blob/65962e27febc3883a17e651cdb23e783668c996f/LICENSE" + "https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language\u003dc#licensing-of-applications-using-gstreamer" ] }, { - "reference": "./LGPL-3.0-linking-exception.json", + "reference": "./PS-or-PDF-font-exception-20170817.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./LGPL-3.0-linking-exception.html", + "detailsUrl": "./PS-or-PDF-font-exception-20170817.html", "referenceNumber": 30, - "name": "LGPL-3.0 Linking Exception", - "licenseExceptionId": "LGPL-3.0-linking-exception", + "name": "PS/PDF font exception (2017-08-17)", + "licenseExceptionId": "PS-or-PDF-font-exception-20170817", "seeAlso": [ - "https://raw.githubusercontent.com/go-xmlpath/xmlpath/v2/LICENSE", - "https://github.com/goamz/goamz/blob/master/LICENSE", - "https://github.com/juju/errors/blob/master/LICENSE" + "https://github.com/ArtifexSoftware/urw-base35-fonts/blob/65962e27febc3883a17e651cdb23e783668c996f/LICENSE" ] }, { - "reference": "./Fawkes-Runtime-exception.json", + "reference": "./Universal-FOSS-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Fawkes-Runtime-exception.html", + "detailsUrl": "./Universal-FOSS-exception-1.0.html", "referenceNumber": 31, - "name": "Fawkes Runtime Exception", - "licenseExceptionId": "Fawkes-Runtime-exception", + "name": "Universal FOSS Exception, Version 1.0", + "licenseExceptionId": "Universal-FOSS-exception-1.0", "seeAlso": [ - "http://www.fawkesrobotics.org/about/license/" + "https://oss.oracle.com/licenses/universal-foss-exception/" ] }, { - "reference": "./Font-exception-2.0.json", - "isDeprecatedLicenseId": false, - "detailsUrl": "./Font-exception-2.0.html", + "reference": "./Nokia-Qt-exception-1.1.json", + "isDeprecatedLicenseId": true, + "detailsUrl": "./Nokia-Qt-exception-1.1.html", "referenceNumber": 32, - "name": "Font exception 2.0", - "licenseExceptionId": "Font-exception-2.0", + "name": "Nokia Qt LGPL exception 1.1", + "licenseExceptionId": "Nokia-Qt-exception-1.1", "seeAlso": [ - "http://www.gnu.org/licenses/gpl-faq.html#FontException" + "https://www.keepassx.org/dev/projects/keepassx/repository/revisions/b8dfb9cc4d5133e0f09cd7533d15a4f1c19a40f2/entry/LICENSE.NOKIA-LGPL-EXCEPTION" ] }, { - "reference": "./Swift-exception.json", + "reference": "./SHL-2.1.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Swift-exception.html", + "detailsUrl": "./SHL-2.1.html", "referenceNumber": 33, - "name": "Swift Exception", - "licenseExceptionId": "Swift-exception", + "name": "Solderpad Hardware License v2.1", + "licenseExceptionId": "SHL-2.1", "seeAlso": [ - "https://swift.org/LICENSE.txt", - "https://github.com/apple/swift-package-manager/blob/7ab2275f447a5eb37497ed63a9340f8a6d1e488b/LICENSE.txt#L205" + "https://solderpad.org/licenses/SHL-2.1/" ] }, { - "reference": "./Bison-exception-2.2.json", + "reference": "./Font-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Bison-exception-2.2.html", + "detailsUrl": "./Font-exception-2.0.html", "referenceNumber": 34, - "name": "Bison exception 2.2", - "licenseExceptionId": "Bison-exception-2.2", + "name": "Font exception 2.0", + "licenseExceptionId": "Font-exception-2.0", "seeAlso": [ - "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" + "http://www.gnu.org/licenses/gpl-faq.html#FontException" ] }, { - "reference": "./Qt-GPL-exception-1.0.json", + "reference": "./Bootloader-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Qt-GPL-exception-1.0.html", + "detailsUrl": "./Bootloader-exception.html", "referenceNumber": 35, - "name": "Qt GPL exception 1.0", - "licenseExceptionId": "Qt-GPL-exception-1.0", + "name": "Bootloader Distribution Exception", + "licenseExceptionId": "Bootloader-exception", "seeAlso": [ - "http://code.qt.io/cgit/qt/qtbase.git/tree/LICENSE.GPL3-EXCEPT" + "https://github.com/pyinstaller/pyinstaller/blob/develop/COPYING.txt" ] }, { - "reference": "./Qt-LGPL-exception-1.1.json", + "reference": "./OpenJDK-assembly-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Qt-LGPL-exception-1.1.html", + "detailsUrl": "./OpenJDK-assembly-exception-1.0.html", "referenceNumber": 36, - "name": "Qt LGPL exception 1.1", - "licenseExceptionId": "Qt-LGPL-exception-1.1", + "name": "OpenJDK Assembly exception 1.0", + "licenseExceptionId": "OpenJDK-assembly-exception-1.0", "seeAlso": [ - "http://code.qt.io/cgit/qt/qtbase.git/tree/LGPL_EXCEPTION.txt" + "http://openjdk.java.net/legal/assembly-exception.html" ] }, { - "reference": "./Nokia-Qt-exception-1.1.json", - "isDeprecatedLicenseId": true, - "detailsUrl": "./Nokia-Qt-exception-1.1.html", + "reference": "./Swift-exception.json", + "isDeprecatedLicenseId": false, + "detailsUrl": "./Swift-exception.html", "referenceNumber": 37, - "name": "Nokia Qt LGPL exception 1.1", - "licenseExceptionId": "Nokia-Qt-exception-1.1", + "name": "Swift Exception", + "licenseExceptionId": "Swift-exception", "seeAlso": [ - "https://www.keepassx.org/dev/projects/keepassx/repository/revisions/b8dfb9cc4d5133e0f09cd7533d15a4f1c19a40f2/entry/LICENSE.NOKIA-LGPL-EXCEPTION" + "https://swift.org/LICENSE.txt", + "https://github.com/apple/swift-package-manager/blob/7ab2275f447a5eb37497ed63a9340f8a6d1e488b/LICENSE.txt#L205" ] }, { - "reference": "./389-exception.json", + "reference": "./GCC-exception-3.1.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./389-exception.html", + "detailsUrl": "./GCC-exception-3.1.html", "referenceNumber": 38, - "name": "389 Directory Server Exception", - "licenseExceptionId": "389-exception", + "name": "GCC Runtime Library exception 3.1", + "licenseExceptionId": "GCC-exception-3.1", "seeAlso": [ - "http://directory.fedoraproject.org/wiki/GPL_Exception_License_Text" + "http://www.gnu.org/licenses/gcc-exception-3.1.html" ] }, { - "reference": "./openvpn-openssl-exception.json", + "reference": "./Linux-syscall-note.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./openvpn-openssl-exception.html", + "detailsUrl": "./Linux-syscall-note.html", "referenceNumber": 39, - "name": "OpenVPN OpenSSL Exception", - "licenseExceptionId": "openvpn-openssl-exception", + "name": "Linux Syscall Note", + "licenseExceptionId": "Linux-syscall-note", "seeAlso": [ - "http://openvpn.net/index.php/license.html" + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/COPYING" ] }, { - "reference": "./freertos-exception-2.0.json", + "reference": "./Autoconf-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./freertos-exception-2.0.html", + "detailsUrl": "./Autoconf-exception-2.0.html", "referenceNumber": 40, - "name": "FreeRTOS Exception 2.0", - "licenseExceptionId": "freertos-exception-2.0", + "name": "Autoconf exception 2.0", + "licenseExceptionId": "Autoconf-exception-2.0", "seeAlso": [ - "https://web.archive.org/web/20060809182744/http://www.freertos.org/a00114.html" + "http://ac-archive.sourceforge.net/doc/copyright.html", + "http://ftp.gnu.org/gnu/autoconf/autoconf-2.59.tar.gz" ] }, { - "reference": "./Autoconf-exception-2.0.json", + "reference": "./GPL-3.0-linking-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Autoconf-exception-2.0.html", + "detailsUrl": "./GPL-3.0-linking-exception.html", "referenceNumber": 41, - "name": "Autoconf exception 2.0", - "licenseExceptionId": "Autoconf-exception-2.0", + "name": "GPL-3.0 Linking Exception", + "licenseExceptionId": "GPL-3.0-linking-exception", "seeAlso": [ - "http://ac-archive.sourceforge.net/doc/copyright.html", - "http://ftp.gnu.org/gnu/autoconf/autoconf-2.59.tar.gz" + "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs" + ] + }, + { + "reference": "./Qt-LGPL-exception-1.1.json", + "isDeprecatedLicenseId": false, + "detailsUrl": "./Qt-LGPL-exception-1.1.html", + "referenceNumber": 42, + "name": "Qt LGPL exception 1.1", + "licenseExceptionId": "Qt-LGPL-exception-1.1", + "seeAlso": [ + "http://code.qt.io/cgit/qt/qtbase.git/tree/LGPL_EXCEPTION.txt" + ] + }, + { + "reference": "./OCCT-exception-1.0.json", + "isDeprecatedLicenseId": false, + "detailsUrl": "./OCCT-exception-1.0.html", + "referenceNumber": 43, + "name": "Open CASCADE Exception 1.0", + "licenseExceptionId": "OCCT-exception-1.0", + "seeAlso": [ + "http://www.opencascade.com/content/licensing" + ] + }, + { + "reference": "./LZMA-exception.json", + "isDeprecatedLicenseId": false, + "detailsUrl": "./LZMA-exception.html", + "referenceNumber": 44, + "name": "LZMA exception", + "licenseExceptionId": "LZMA-exception", + "seeAlso": [ + "http://nsis.sourceforge.net/Docs/AppendixI.html#I.6" ] } ], - "releaseDate": "2021-11-19" + "releaseDate": "2022-10-07" } \ No newline at end of file diff --git a/src/scanoss/data/spdx-licenses.json b/src/scanoss/data/spdx-licenses.json index 2be785d5..7e342897 100644 --- a/src/scanoss/data/spdx-licenses.json +++ b/src/scanoss/data/spdx-licenses.json @@ -1,3258 +1,3247 @@ { - "licenseListVersion": "f9911cd", + "licenseListVersion": "03c58ca", "licenses": [ { - "reference": "https://spdx.org/licenses/EUDatagrid.html", + "reference": "https://spdx.org/licenses/xpp.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUDatagrid.json", + "detailsUrl": "https://spdx.org/licenses/xpp.json", "referenceNumber": 0, - "name": "EU DataGrid Software License", - "licenseId": "EUDatagrid", + "name": "XPP License", + "licenseId": "xpp", "seeAlso": [ - "http://eu-datagrid.web.cern.ch/eu-datagrid/license.html", - "https://opensource.org/licenses/EUDatagrid" + "https://fedoraproject.org/wiki/Licensing/xpp" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/VOSTROM.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/VOSTROM.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-only.json", "referenceNumber": 1, - "name": "VOSTROM Public License for Open Source", - "licenseId": "VOSTROM", + "name": "GNU Free Documentation License v1.1 only", + "licenseId": "GFDL-1.1-only", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/VOSTROM" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.json", + "reference": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.json", "referenceNumber": 2, - "name": "GNU Free Documentation License v1.3 only - no invariants", - "licenseId": "GFDL-1.3-no-invariants-only", + "name": "GNU General Public License v2.0 w/Autoconf exception", + "licenseId": "GPL-2.0-with-autoconf-exception", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "http://ac-archive.sourceforge.net/doc/copyright.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-1.0.html", + "reference": "https://spdx.org/licenses/copyleft-next-0.3.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-1.0.json", + "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.1.json", "referenceNumber": 3, - "name": "Creative Commons Attribution Share Alike 1.0 Generic", - "licenseId": "CC-BY-SA-1.0", + "name": "copyleft-next 0.3.1", + "licenseId": "copyleft-next-0.3.1", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/1.0/legalcode" + "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-DE.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-DE.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.5.json", "referenceNumber": 4, - "name": "Creative Commons Attribution 3.0 Germany", - "licenseId": "CC-BY-3.0-DE", + "name": "Creative Commons Attribution Non Commercial 2.5 Generic", + "licenseId": "CC-BY-NC-2.5", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/de/legalcode" + "https://creativecommons.org/licenses/by-nc/2.5/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-2.5.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.5.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json", "referenceNumber": 5, - "name": "Creative Commons Attribution Share Alike 2.5 Generic", - "licenseId": "CC-BY-SA-2.5", + "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + "licenseId": "CC-BY-NC-SA-3.0-IGO", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/2.5/legalcode" + "https://creativecommons.org/licenses/by-nc-sa/3.0/igo/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/HaskellReport.html", + "reference": "https://spdx.org/licenses/QPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/HaskellReport.json", + "detailsUrl": "https://spdx.org/licenses/QPL-1.0.json", "referenceNumber": 6, - "name": "Haskell Language Report License", - "licenseId": "HaskellReport", + "name": "Q Public License 1.0", + "licenseId": "QPL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Haskell_Language_Report_License" + "http://doc.qt.nokia.com/3.3/license.html", + "https://opensource.org/licenses/QPL-1.0", + "https://doc.qt.io/archives/3.3/license.html" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SimPL-2.0.html", + "reference": "https://spdx.org/licenses/ErlPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SimPL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/ErlPL-1.1.json", "referenceNumber": 7, - "name": "Simple Public License 2.0", - "licenseId": "SimPL-2.0", + "name": "Erlang Public License v1.1", + "licenseId": "ErlPL-1.1", "seeAlso": [ - "https://opensource.org/licenses/SimPL-2.0" + "http://www.erlang.org/EPLICENSE" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ClArtistic.html", + "reference": "https://spdx.org/licenses/Wsuipa.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ClArtistic.json", + "detailsUrl": "https://spdx.org/licenses/Wsuipa.json", "referenceNumber": 8, - "name": "Clarified Artistic License", - "licenseId": "ClArtistic", + "name": "Wsuipa License", + "licenseId": "Wsuipa", "seeAlso": [ - "http://gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/", - "http://www.ncftp.com/ncftp/doc/LICENSE.txt" + "https://fedoraproject.org/wiki/Licensing/Wsuipa" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Nunit.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/Nunit.json", + "reference": "https://spdx.org/licenses/Naumen.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Naumen.json", "referenceNumber": 9, - "name": "Nunit License", - "licenseId": "Nunit", + "name": "Naumen Public License", + "licenseId": "Naumen", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Nunit" + "https://opensource.org/licenses/Naumen" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/SMPPL.html", + "reference": "https://spdx.org/licenses/FSFUL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SMPPL.json", + "detailsUrl": "https://spdx.org/licenses/FSFUL.json", "referenceNumber": 10, - "name": "Secure Messaging Protocol Public License", - "licenseId": "SMPPL", + "name": "FSF Unlimited License", + "licenseId": "FSFUL", "seeAlso": [ - "https://github.com/dcblake/SMP/blob/master/Documentation/License.txt" + "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0+.json", + "reference": "https://spdx.org/licenses/AGPL-3.0-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-or-later.json", "referenceNumber": 11, - "name": "GNU Library General Public License v2 or later", - "licenseId": "LGPL-2.0+", + "name": "GNU Affero General Public License v3.0 or later", + "licenseId": "AGPL-3.0-or-later", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + "https://www.gnu.org/licenses/agpl.txt", + "https://opensource.org/licenses/AGPL-3.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CUA-OPL-1.0.html", + "reference": "https://spdx.org/licenses/LGPL-3.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CUA-OPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-or-later.json", "referenceNumber": 12, - "name": "CUA Office Public License v1.0", - "licenseId": "CUA-OPL-1.0", + "name": "GNU Lesser General Public License v3.0 or later", + "licenseId": "LGPL-3.0-or-later", "seeAlso": [ - "https://opensource.org/licenses/CUA-OPL-1.0" + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-1.3.html", + "reference": "https://spdx.org/licenses/NTP-0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.3.json", + "detailsUrl": "https://spdx.org/licenses/NTP-0.json", "referenceNumber": 13, - "name": "Open LDAP Public License v1.3", - "licenseId": "OLDAP-1.3", + "name": "NTP No Attribution", + "licenseId": "NTP-0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003de5f8117f0ce088d0bd7a8e18ddf37eaa40eb09b1" + "https://github.com/tytso/e2fsprogs/blob/master/lib/et/et_name.c" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Giftware.html", + "reference": "https://spdx.org/licenses/NAIST-2003.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Giftware.json", + "detailsUrl": "https://spdx.org/licenses/NAIST-2003.json", "referenceNumber": 14, - "name": "Giftware License", - "licenseId": "Giftware", + "name": "Nara Institute of Science and Technology License (2003)", + "licenseId": "NAIST-2003", "seeAlso": [ - "http://liballeg.org/license.html#allegro-4-the-giftware-license" + "https://enterprise.dejacode.com/licenses/public/naist-2003/#license-text", + "https://github.com/nodejs/node/blob/4a19cc8947b1bba2b2d27816ec3d0edf9b28e503/LICENSE#L343" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BitTorrent-1.1.html", + "reference": "https://spdx.org/licenses/Newsletr.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.1.json", + "detailsUrl": "https://spdx.org/licenses/Newsletr.json", "referenceNumber": 15, - "name": "BitTorrent Open Source License v1.1", - "licenseId": "BitTorrent-1.1", + "name": "Newsletr License", + "licenseId": "Newsletr", "seeAlso": [ - "http://directory.fsf.org/wiki/License:BitTorrentOSL1.1" + "https://fedoraproject.org/wiki/Licensing/Newsletr" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ISC.html", + "reference": "https://spdx.org/licenses/MPL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ISC.json", + "detailsUrl": "https://spdx.org/licenses/MPL-2.0.json", "referenceNumber": 16, - "name": "ISC License", - "licenseId": "ISC", + "name": "Mozilla Public License 2.0", + "licenseId": "MPL-2.0", "seeAlso": [ - "https://www.isc.org/licenses/", - "https://www.isc.org/downloads/software-support-policy/isc-license/", - "https://opensource.org/licenses/ISC" + "https://www.mozilla.org/MPL/2.0/", + "https://opensource.org/licenses/MPL-2.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LPPL-1.1.html", + "reference": "https://spdx.org/licenses/OLDAP-2.0.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.1.json", "referenceNumber": 17, - "name": "LaTeX Project Public License v1.1", - "licenseId": "LPPL-1.1", + "name": "Open LDAP Public License v2.0.1", + "licenseId": "OLDAP-2.0.1", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-1.txt" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db6d68acd14e51ca3aab4428bf26522aa74873f0e" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ECL-2.0.html", + "reference": "https://spdx.org/licenses/Giftware.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ECL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Giftware.json", "referenceNumber": 18, - "name": "Educational Community License v2.0", - "licenseId": "ECL-2.0", + "name": "Giftware License", + "licenseId": "Giftware", "seeAlso": [ - "https://opensource.org/licenses/ECL-2.0" + "http://liballeg.org/license.html#allegro-4-the-giftware-license" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Entessa.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Entessa.json", + "reference": "https://spdx.org/licenses/AGPL-1.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/AGPL-1.0.json", "referenceNumber": 19, - "name": "Entessa Public License v1.0", - "licenseId": "Entessa", + "name": "Affero General Public License v1.0", + "licenseId": "AGPL-1.0", "seeAlso": [ - "https://opensource.org/licenses/Entessa" + "http://www.affero.org/oagpl.html" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Spencer-86.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Spencer-86.json", + "reference": "https://spdx.org/licenses/GPL-1.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-1.0+.json", "referenceNumber": 20, - "name": "Spencer License 86", - "licenseId": "Spencer-86", + "name": "GNU General Public License v1.0 or later", + "licenseId": "GPL-1.0+", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/O-UDA-1.0.html", + "reference": "https://spdx.org/licenses/Eurosym.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/O-UDA-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Eurosym.json", "referenceNumber": 21, - "name": "Open Use of Data Agreement v1.0", - "licenseId": "O-UDA-1.0", + "name": "Eurosym License", + "licenseId": "Eurosym", "seeAlso": [ - "https://github.com/microsoft/Open-Use-of-Data-Agreement/blob/v1.0/O-UDA-1.0.md", - "https://cdla.dev/open-use-of-data-agreement-v1-0/" + "https://fedoraproject.org/wiki/Licensing/Eurosym" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-4-Clause.html", + "reference": "https://spdx.org/licenses/EPICS.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause.json", + "detailsUrl": "https://spdx.org/licenses/EPICS.json", "referenceNumber": 22, - "name": "BSD 4-Clause \"Original\" or \"Old\" License", - "licenseId": "BSD-4-Clause", + "name": "EPICS Open License", + "licenseId": "EPICS", "seeAlso": [ - "http://directory.fsf.org/wiki/License:BSD_4Clause" + "https://epics.anl.gov/license/open.php" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Condor-1.1.html", + "reference": "https://spdx.org/licenses/OLDAP-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Condor-1.1.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.json", "referenceNumber": 23, - "name": "Condor Public License v1.1", - "licenseId": "Condor-1.1", + "name": "Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B)", + "licenseId": "OLDAP-2.0", "seeAlso": [ - "http://research.cs.wisc.edu/condor/license.html#condor", - "http://web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcbf50f4e1185a21abd4c0a54d3f4341fe28f36ea" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ODC-By-1.0.html", + "reference": "https://spdx.org/licenses/BSD-Protection.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ODC-By-1.0.json", + "detailsUrl": "https://spdx.org/licenses/BSD-Protection.json", "referenceNumber": 24, - "name": "Open Data Commons Attribution License v1.0", - "licenseId": "ODC-By-1.0", + "name": "BSD Protection License", + "licenseId": "BSD-Protection", "seeAlso": [ - "https://opendatacommons.org/licenses/by/1.0/" + "https://fedoraproject.org/wiki/Licensing/BSD_Protection_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/IPA.html", + "reference": "https://spdx.org/licenses/SAX-PD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IPA.json", + "detailsUrl": "https://spdx.org/licenses/SAX-PD.json", "referenceNumber": 25, - "name": "IPA Font License", - "licenseId": "IPA", + "name": "Sax Public Domain Notice", + "licenseId": "SAX-PD", "seeAlso": [ - "https://opensource.org/licenses/IPA" + "http://www.saxproject.org/copying.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Net-SNMP.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Net-SNMP.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.json", "referenceNumber": 26, - "name": "Net-SNMP License", - "licenseId": "Net-SNMP", + "name": "Creative Commons Attribution Non Commercial 3.0 Germany", + "licenseId": "CC-BY-NC-3.0-DE", "seeAlso": [ - "http://net-snmp.sourceforge.net/about/license.html" + "https://creativecommons.org/licenses/by-nc/3.0/de/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-4-Clause-Shortened.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-Shortened.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.json", "referenceNumber": 27, - "name": "BSD 4 Clause Shortened", - "licenseId": "BSD-4-Clause-Shortened", + "name": "GNU Free Documentation License v1.3 only - no invariants", + "licenseId": "GFDL-1.3-no-invariants-only", "seeAlso": [ - "https://metadata.ftp-master.debian.org/changelogs//main/a/arpwatch/arpwatch_2.1a15-7_copyright" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDLA-Permissive-2.0.html", + "reference": "https://spdx.org/licenses/HPND.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-2.0.json", + "detailsUrl": "https://spdx.org/licenses/HPND.json", "referenceNumber": 28, - "name": "Community Data License Agreement Permissive 2.0", - "licenseId": "CDLA-Permissive-2.0", + "name": "Historical Permission Notice and Disclaimer", + "licenseId": "HPND", "seeAlso": [ - "https://cdla.dev/permissive-2-0" + "https://opensource.org/licenses/HPND" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.json", "referenceNumber": 29, - "name": "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", - "licenseId": "CC-BY-NC-ND-2.5", + "name": "Creative Commons Attribution Share Alike 3.0 Germany", + "licenseId": "CC-BY-SA-3.0-DE", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/2.5/legalcode" + "https://creativecommons.org/licenses/by-sa/3.0/de/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0.json", "referenceNumber": 30, - "name": "BSD 3-Clause No Nuclear License", - "licenseId": "BSD-3-Clause-No-Nuclear-License", + "name": "Creative Commons Attribution Share Alike 3.0 Unported", + "licenseId": "CC-BY-SA-3.0", "seeAlso": [ - "http://download.oracle.com/otn-pub/java/licenses/bsd.txt?AuthParam\u003d1467140197_43d516ce1776bd08a58235a7785be1cc" + "https://creativecommons.org/licenses/by-sa/3.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-NL.html", + "reference": "https://spdx.org/licenses/OLDAP-2.4.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-NL.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.4.json", "referenceNumber": 31, - "name": "Creative Commons Attribution 3.0 Netherlands", - "licenseId": "CC-BY-3.0-NL", + "name": "Open LDAP Public License v2.4", + "licenseId": "OLDAP-2.4", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/nl/legalcode" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcd1284c4a91a8a380d904eee68d1583f989ed386" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Latex2e.html", + "reference": "https://spdx.org/licenses/Zimbra-1.4.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Latex2e.json", + "detailsUrl": "https://spdx.org/licenses/Zimbra-1.4.json", "referenceNumber": 32, - "name": "Latex2e License", - "licenseId": "Latex2e", + "name": "Zimbra Public License v1.4", + "licenseId": "Zimbra-1.4", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Latex2e" + "http://www.zimbra.com/legal/zimbra-public-license-1-4" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGL-Canada-2.0.html", + "reference": "https://spdx.org/licenses/PostgreSQL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-Canada-2.0.json", + "detailsUrl": "https://spdx.org/licenses/PostgreSQL.json", "referenceNumber": 33, - "name": "Open Government Licence - Canada", - "licenseId": "OGL-Canada-2.0", + "name": "PostgreSQL License", + "licenseId": "PostgreSQL", "seeAlso": [ - "https://open.canada.ca/en/open-government-licence-canada" + "http://www.postgresql.org/about/licence", + "https://opensource.org/licenses/PostgreSQL" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-font-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-font-exception.json", + "reference": "https://spdx.org/licenses/Noweb.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Noweb.json", "referenceNumber": 34, - "name": "GNU General Public License v2.0 w/Font exception", - "licenseId": "GPL-2.0-with-font-exception", + "name": "Noweb License", + "licenseId": "Noweb", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-faq.html#FontException" + "https://fedoraproject.org/wiki/Licensing/Noweb" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ZPL-2.1.html", + "reference": "https://spdx.org/licenses/NIST-PD-fallback.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ZPL-2.1.json", + "detailsUrl": "https://spdx.org/licenses/NIST-PD-fallback.json", "referenceNumber": 35, - "name": "Zope Public License 2.1", - "licenseId": "ZPL-2.1", + "name": "NIST Public Domain Notice with license fallback", + "licenseId": "NIST-PD-fallback", "seeAlso": [ - "http://old.zope.org/Resources/ZPL/" + "https://github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE", + "https://github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Modification.html", + "reference": "https://spdx.org/licenses/Mup.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Modification.json", + "detailsUrl": "https://spdx.org/licenses/Mup.json", "referenceNumber": 36, - "name": "BSD 3-Clause Modification", - "licenseId": "BSD-3-Clause-Modification", + "name": "Mup License", + "licenseId": "Mup", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:BSD#Modification_Variant" + "https://fedoraproject.org/wiki/Licensing/Mup" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/VSL-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/VSL-1.0.json", + "reference": "https://spdx.org/licenses/GPL-3.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0.json", "referenceNumber": 37, - "name": "Vovida Software License v1.0", - "licenseId": "VSL-1.0", + "name": "GNU General Public License v3.0 only", + "licenseId": "GPL-3.0", "seeAlso": [ - "https://opensource.org/licenses/VSL-1.0" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/RSA-MD.html", + "reference": "https://spdx.org/licenses/checkmk.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RSA-MD.json", + "detailsUrl": "https://spdx.org/licenses/checkmk.json", "referenceNumber": 38, - "name": "RSA Message-Digest License", - "licenseId": "RSA-MD", + "name": "Checkmk License", + "licenseId": "checkmk", "seeAlso": [ - "http://www.faqs.org/rfcs/rfc1321.html" + "https://github.com/libcheck/check/blob/master/checkmk/checkmk.in" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDL-1.0.html", + "reference": "https://spdx.org/licenses/Parity-6.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Parity-6.0.0.json", "referenceNumber": 39, - "name": "Common Documentation License 1.0", - "licenseId": "CDL-1.0", + "name": "The Parity Public License 6.0.0", + "licenseId": "Parity-6.0.0", "seeAlso": [ - "http://www.opensource.apple.com/cdl/", - "https://fedoraproject.org/wiki/Licensing/Common_Documentation_License", - "https://www.gnu.org/licenses/license-list.html#ACDL" + "https://paritylicense.com/versions/6.0.0.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Parity-7.0.0.html", + "reference": "https://spdx.org/licenses/Linux-OpenIB.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Parity-7.0.0.json", + "detailsUrl": "https://spdx.org/licenses/Linux-OpenIB.json", "referenceNumber": 40, - "name": "The Parity Public License 7.0.0", - "licenseId": "Parity-7.0.0", + "name": "Linux Kernel Variant of OpenIB.org license", + "licenseId": "Linux-OpenIB", "seeAlso": [ - "https://paritylicense.com/versions/7.0.0.html" + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.html", + "reference": "https://spdx.org/licenses/TCL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.json", + "detailsUrl": "https://spdx.org/licenses/TCL.json", "referenceNumber": 41, - "name": "Creative Commons Attribution Share Alike 2.0 England and Wales", - "licenseId": "CC-BY-SA-2.0-UK", + "name": "TCL/TK License", + "licenseId": "TCL", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/2.0/uk/legalcode" + "http://www.tcl.tk/software/tcltk/license.html", + "https://fedoraproject.org/wiki/Licensing/TCL" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SCEA.html", + "reference": "https://spdx.org/licenses/CERN-OHL-P-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SCEA.json", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-P-2.0.json", "referenceNumber": 42, - "name": "SCEA Shared Source License", - "licenseId": "SCEA", + "name": "CERN Open Hardware Licence Version 2 - Permissive", + "licenseId": "CERN-OHL-P-2.0", "seeAlso": [ - "http://research.scea.com/scea_shared_source_license.html" + "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.html", + "reference": "https://spdx.org/licenses/EPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json", + "detailsUrl": "https://spdx.org/licenses/EPL-1.0.json", "referenceNumber": 43, - "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", - "licenseId": "CC-BY-NC-SA-3.0-DE", + "name": "Eclipse Public License 1.0", + "licenseId": "EPL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/3.0/de/legalcode" + "http://www.eclipse.org/legal/epl-v10.html", + "https://opensource.org/licenses/EPL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SHL-0.5.html", + "reference": "https://spdx.org/licenses/EUPL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SHL-0.5.json", + "detailsUrl": "https://spdx.org/licenses/EUPL-1.2.json", "referenceNumber": 44, - "name": "Solderpad Hardware License v0.5", - "licenseId": "SHL-0.5", + "name": "European Union Public License 1.2", + "licenseId": "EUPL-1.2", "seeAlso": [ - "https://solderpad.org/licenses/SHL-0.5/" + "https://joinup.ec.europa.eu/page/eupl-text-11-12", + "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl_v1.2_en.pdf", + "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/2020-03/EUPL-1.2%20EN.txt", + "https://joinup.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt", + "http://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri\u003dCELEX:32017D0863", + "https://opensource.org/licenses/EUPL-1.2" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause.html", + "reference": "https://spdx.org/licenses/D-FSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause.json", + "detailsUrl": "https://spdx.org/licenses/D-FSL-1.0.json", "referenceNumber": 45, - "name": "BSD 3-Clause \"New\" or \"Revised\" License", - "licenseId": "BSD-3-Clause", + "name": "Deutsche Freie Software Lizenz", + "licenseId": "D-FSL-1.0", "seeAlso": [ - "https://opensource.org/licenses/BSD-3-Clause" + "http://www.dipp.nrw.de/d-fsl/lizenzen/", + "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/de/D-FSL-1_0_de.txt", + "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/en/D-FSL-1_0_en.txt", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/deutsche-freie-software-lizenz", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/german-free-software-license", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_de.txt/at_download/file", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_en.txt/at_download/file" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NLOD-2.0.html", + "reference": "https://spdx.org/licenses/Rdisc.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NLOD-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Rdisc.json", "referenceNumber": 46, - "name": "Norwegian Licence for Open Government Data (NLOD) 2.0", - "licenseId": "NLOD-2.0", + "name": "Rdisc License", + "licenseId": "Rdisc", "seeAlso": [ - "http://data.norge.no/nlod/en/2.0" + "https://fedoraproject.org/wiki/Licensing/Rdisc_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Saxpath.html", + "reference": "https://spdx.org/licenses/OLDAP-2.6.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Saxpath.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.6.json", "referenceNumber": 47, - "name": "Saxpath License", - "licenseId": "Saxpath", + "name": "Open LDAP Public License v2.6", + "licenseId": "OLDAP-2.6", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Saxpath_License" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d1cae062821881f41b73012ba816434897abf4205" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/APAFML.html", + "reference": "https://spdx.org/licenses/Artistic-1.0-cl8.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APAFML.json", + "detailsUrl": "https://spdx.org/licenses/Artistic-1.0-cl8.json", "referenceNumber": 48, - "name": "Adobe Postscript AFM License", - "licenseId": "APAFML", + "name": "Artistic License 1.0 w/clause 8", + "licenseId": "Artistic-1.0-cl8", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AdobePostscriptAFM" + "https://opensource.org/licenses/Artistic-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Info-ZIP.html", + "reference": "https://spdx.org/licenses/SNIA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Info-ZIP.json", + "detailsUrl": "https://spdx.org/licenses/SNIA.json", "referenceNumber": 49, - "name": "Info-ZIP License", - "licenseId": "Info-ZIP", + "name": "SNIA Public License 1.1", + "licenseId": "SNIA", "seeAlso": [ - "http://www.info-zip.org/license.html" + "https://fedoraproject.org/wiki/Licensing/SNIA_Public_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/APSL-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.json", "referenceNumber": 50, - "name": "Apple Public Source License 1.1", - "licenseId": "APSL-1.1", + "name": "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + "licenseId": "CC-BY-NC-SA-4.0", "seeAlso": [ - "http://www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE" + "https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/HTMLTIDY.html", + "reference": "https://spdx.org/licenses/ICU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/HTMLTIDY.json", + "detailsUrl": "https://spdx.org/licenses/ICU.json", "referenceNumber": 51, - "name": "HTML Tidy License", - "licenseId": "HTMLTIDY", + "name": "ICU License", + "licenseId": "ICU", "seeAlso": [ - "https://github.com/htacg/tidy-html5/blob/next/README/LICENSE.md" + "http://source.icu-project.org/repos/icu/icu/trunk/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/COIL-1.0.html", + "reference": "https://spdx.org/licenses/FSFAP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/COIL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/FSFAP.json", "referenceNumber": 52, - "name": "Copyfree Open Innovation License", - "licenseId": "COIL-1.0", + "name": "FSF All Permissive License", + "licenseId": "FSFAP", "seeAlso": [ - "https://coil.apotheon.org/plaintext/01.0.txt" + "https://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.html", + "reference": "https://spdx.org/licenses/MIT-Modern-Variant.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.json", + "detailsUrl": "https://spdx.org/licenses/MIT-Modern-Variant.json", "referenceNumber": 53, - "name": "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", - "licenseId": "CC-BY-NC-SA-1.0", + "name": "MIT License Modern Variant", + "licenseId": "MIT-Modern-Variant", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/1.0/legalcode" + "https://fedoraproject.org/wiki/Licensing:MIT#Modern_Variants", + "https://ptolemy.berkeley.edu/copyright.htm", + "https://pirlwww.lpl.arizona.edu/resources/guide/software/PerlTk/Tixlic.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-2.0.html", + "reference": "https://spdx.org/licenses/JSON.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.0.json", + "detailsUrl": "https://spdx.org/licenses/JSON.json", "referenceNumber": 54, - "name": "Creative Commons Attribution Non Commercial 2.0 Generic", - "licenseId": "CC-BY-NC-2.0", + "name": "JSON License", + "licenseId": "JSON", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/2.0/legalcode" + "http://www.json.org/license.html" ], "isOsiApproved": false, "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/SSH-short.html", + "reference": "https://spdx.org/licenses/LZMA-SDK-9.11-to-9.20.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SSH-short.json", + "detailsUrl": "https://spdx.org/licenses/LZMA-SDK-9.11-to-9.20.json", "referenceNumber": 55, - "name": "SSH short notice", - "licenseId": "SSH-short", + "name": "LZMA SDK License (versions 9.11 to 9.20)", + "licenseId": "LZMA-SDK-9.11-to-9.20", "seeAlso": [ - "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h", - "http://web.mit.edu/kolya/.f/root/athena.mit.edu/sipb.mit.edu/project/openssh/OldFiles/src/openssh-2.9.9p2/ssh-add.1", - "https://joinup.ec.europa.eu/svn/lesoll/trunk/italc/lib/src/dsa_key.cpp" + "https://www.7-zip.org/sdk.html", + "https://sourceforge.net/projects/sevenzip/files/LZMA%20SDK/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Mup.html", + "reference": "https://spdx.org/licenses/0BSD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Mup.json", + "detailsUrl": "https://spdx.org/licenses/0BSD.json", "referenceNumber": 56, - "name": "Mup License", - "licenseId": "Mup", + "name": "BSD Zero Clause License", + "licenseId": "0BSD", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Mup" + "http://landley.net/toybox/license.html", + "https://opensource.org/licenses/0BSD" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/EFL-1.0.html", + "reference": "https://spdx.org/licenses/LPPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EFL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.0.json", "referenceNumber": 57, - "name": "Eiffel Forum License v1.0", - "licenseId": "EFL-1.0", + "name": "LaTeX Project Public License v1.0", + "licenseId": "LPPL-1.0", "seeAlso": [ - "http://www.eiffel-nice.org/license/forum.txt", - "https://opensource.org/licenses/EFL-1.0" + "http://www.latex-project.org/lppl/lppl-1-0.txt" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-3.0-only.html", + "reference": "https://spdx.org/licenses/mpich2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-only.json", + "detailsUrl": "https://spdx.org/licenses/mpich2.json", "referenceNumber": 58, - "name": "GNU Lesser General Public License v3.0 only", - "licenseId": "LGPL-3.0-only", + "name": "mpich2 License", + "licenseId": "mpich2", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://opensource.org/licenses/LGPL-3.0" + "https://fedoraproject.org/wiki/Licensing/MIT" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NPOSL-3.0.html", + "reference": "https://spdx.org/licenses/TAPR-OHL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NPOSL-3.0.json", + "detailsUrl": "https://spdx.org/licenses/TAPR-OHL-1.0.json", "referenceNumber": 59, - "name": "Non-Profit Open Software License 3.0", - "licenseId": "NPOSL-3.0", + "name": "TAPR Open Hardware License v1.0", + "licenseId": "TAPR-OHL-1.0", "seeAlso": [ - "https://opensource.org/licenses/NOSL3.0" + "https://www.tapr.org/OHL" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-4.0.html", + "reference": "https://spdx.org/licenses/blessing.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-4.0.json", + "detailsUrl": "https://spdx.org/licenses/blessing.json", "referenceNumber": 60, - "name": "Creative Commons Attribution No Derivatives 4.0 International", - "licenseId": "CC-BY-ND-4.0", + "name": "SQLite Blessing", + "licenseId": "blessing", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/4.0/legalcode" + "https://www.sqlite.org/src/artifact/e33a4df7e32d742a?ln\u003d4-9", + "https://sqlite.org/src/artifact/df5091916dbb40e6" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Xerox.html", + "reference": "https://spdx.org/licenses/Aladdin.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Xerox.json", + "detailsUrl": "https://spdx.org/licenses/Aladdin.json", "referenceNumber": 61, - "name": "Xerox License", - "licenseId": "Xerox", + "name": "Aladdin Free Public License", + "licenseId": "Aladdin", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Xerox" + "http://pages.cs.wisc.edu/~ghost/doc/AFPL/6.01/Public.htm" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/OGL-UK-3.0.html", + "reference": "https://spdx.org/licenses/MIT-advertising.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-UK-3.0.json", + "detailsUrl": "https://spdx.org/licenses/MIT-advertising.json", "referenceNumber": 62, - "name": "Open Government Licence v3.0", - "licenseId": "OGL-UK-3.0", + "name": "Enlightenment License (e16)", + "licenseId": "MIT-advertising", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" + "https://fedoraproject.org/wiki/Licensing/MIT_With_Advertising" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AAL.html", + "reference": "https://spdx.org/licenses/LAL-1.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AAL.json", + "detailsUrl": "https://spdx.org/licenses/LAL-1.3.json", "referenceNumber": 63, - "name": "Attribution Assurance License", - "licenseId": "AAL", + "name": "Licence Art Libre 1.3", + "licenseId": "LAL-1.3", "seeAlso": [ - "https://opensource.org/licenses/attribution" + "https://artlibre.org/" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/QPL-1.0.html", + "reference": "https://spdx.org/licenses/X11.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/QPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/X11.json", "referenceNumber": 64, - "name": "Q Public License 1.0", - "licenseId": "QPL-1.0", + "name": "X11 License", + "licenseId": "X11", "seeAlso": [ - "http://doc.qt.nokia.com/3.3/license.html", - "https://opensource.org/licenses/QPL-1.0", - "https://doc.qt.io/archives/3.3/license.html" + "http://www.xfree86.org/3.3.6/COPYRIGHT2.html#3" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Dotseqn.html", + "reference": "https://spdx.org/licenses/NOSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Dotseqn.json", + "detailsUrl": "https://spdx.org/licenses/NOSL.json", "referenceNumber": 65, - "name": "Dotseqn License", - "licenseId": "Dotseqn", + "name": "Netizen Open Source License", + "licenseId": "NOSL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Dotseqn" + "http://bits.netizen.com.au/licenses/NOSL/nosl.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-2.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0.json", + "reference": "https://spdx.org/licenses/BSD-2-Clause-Patent.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Patent.json", "referenceNumber": 66, - "name": "GNU General Public License v2.0 only", - "licenseId": "GPL-2.0", + "name": "BSD-2-Clause Plus Patent License", + "licenseId": "BSD-2-Clause-Patent", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "https://opensource.org/licenses/BSDplusPatent" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/libselinux-1.0.html", + "reference": "https://spdx.org/licenses/MulanPSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/libselinux-1.0.json", + "detailsUrl": "https://spdx.org/licenses/MulanPSL-1.0.json", "referenceNumber": 67, - "name": "libselinux public domain notice", - "licenseId": "libselinux-1.0", + "name": "Mulan Permissive Software License, Version 1", + "licenseId": "MulanPSL-1.0", "seeAlso": [ - "https://github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE" + "https://license.coscl.org.cn/MulanPSL/", + "https://github.com/yuwenlong/longphp/blob/25dfb70cc2a466dc4bb55ba30901cbce08d164b5/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MIT-advertising.html", + "reference": "https://spdx.org/licenses/Spencer-86.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-advertising.json", + "detailsUrl": "https://spdx.org/licenses/Spencer-86.json", "referenceNumber": 68, - "name": "Enlightenment License (e16)", - "licenseId": "MIT-advertising", + "name": "Spencer License 86", + "licenseId": "Spencer-86", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT_With_Advertising" + "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SSH-OpenSSH.html", + "reference": "https://spdx.org/licenses/Net-SNMP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SSH-OpenSSH.json", + "detailsUrl": "https://spdx.org/licenses/Net-SNMP.json", "referenceNumber": 69, - "name": "SSH OpenSSH license", - "licenseId": "SSH-OpenSSH", + "name": "Net-SNMP License", + "licenseId": "Net-SNMP", "seeAlso": [ - "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10" + "http://net-snmp.sourceforge.net/about/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPLLR.html", + "reference": "https://spdx.org/licenses/LGPL-3.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPLLR.json", + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-only.json", "referenceNumber": 70, - "name": "Lesser General Public License For Linguistic Resources", - "licenseId": "LGPLLR", + "name": "GNU Lesser General Public License v3.0 only", + "licenseId": "LGPL-3.0-only", "seeAlso": [ - "http://www-igm.univ-mlv.fr/~unitex/lgpllr.html" + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/ErlPL-1.1.html", + "reference": "https://spdx.org/licenses/AGPL-1.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ErlPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-or-later.json", "referenceNumber": 71, - "name": "Erlang Public License v1.1", - "licenseId": "ErlPL-1.1", + "name": "Affero General Public License v1.0 or later", + "licenseId": "AGPL-1.0-or-later", "seeAlso": [ - "http://www.erlang.org/EPLICENSE" + "http://www.affero.org/oagpl.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.json", + "reference": "https://spdx.org/licenses/Apache-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Apache-1.0.json", "referenceNumber": 72, - "name": "BSD 2-Clause NetBSD License", - "licenseId": "BSD-2-Clause-NetBSD", + "name": "Apache License 1.0", + "licenseId": "Apache-1.0", "seeAlso": [ - "http://www.netbsd.org/about/redistribution.html#default" + "http://www.apache.org/licenses/LICENSE-1.0" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/TU-Berlin-2.0.html", + "reference": "https://spdx.org/licenses/Cube.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TU-Berlin-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Cube.json", "referenceNumber": 73, - "name": "Technische Universitaet Berlin License 2.0", - "licenseId": "TU-Berlin-2.0", + "name": "Cube License", + "licenseId": "Cube", "seeAlso": [ - "https://github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt" + "https://fedoraproject.org/wiki/Licensing/Cube" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-2.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.0.json", + "reference": "https://spdx.org/licenses/LGPL-3.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0+.json", "referenceNumber": 74, - "name": "Creative Commons Attribution No Derivatives 2.0 Generic", - "licenseId": "CC-BY-ND-2.0", + "name": "GNU Lesser General Public License v3.0 or later", + "licenseId": "LGPL-3.0+", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/2.0/legalcode" + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.html", + "reference": "https://spdx.org/licenses/NCGL-UK-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.json", + "detailsUrl": "https://spdx.org/licenses/NCGL-UK-2.0.json", "referenceNumber": 75, - "name": "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", - "licenseId": "CC-BY-NC-ND-4.0", + "name": "Non-Commercial Government Licence", + "licenseId": "NCGL-UK-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode" + "http://www.nationalarchives.gov.uk/doc/non-commercial-government-licence/version/2/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Plexus.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-AT.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Plexus.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-AT.json", "referenceNumber": 76, - "name": "Plexus Classworlds License", - "licenseId": "Plexus", + "name": "Creative Commons Attribution 3.0 Austria", + "licenseId": "CC-BY-3.0-AT", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Plexus_Classworlds_License" + "https://creativecommons.org/licenses/by/3.0/at/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SPL-1.0.html", + "reference": "https://spdx.org/licenses/OGL-UK-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/OGL-UK-2.0.json", "referenceNumber": 77, - "name": "Sun Public License v1.0", - "licenseId": "SPL-1.0", + "name": "Open Government Licence v2.0", + "licenseId": "OGL-UK-2.0", "seeAlso": [ - "https://opensource.org/licenses/SPL-1.0" + "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.html", + "reference": "https://spdx.org/licenses/CC-BY-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-1.0.json", "referenceNumber": 78, - "name": "BSD 3-Clause No Military License", - "licenseId": "BSD-3-Clause-No-Military-License", + "name": "Creative Commons Attribution 1.0 Generic", + "licenseId": "CC-BY-1.0", "seeAlso": [ - "https://gitlab.syncad.com/hive/dhive/-/blob/master/LICENSE", - "https://github.com/greymass/swift-eosio/blob/master/LICENSE" + "https://creativecommons.org/licenses/by/1.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CPAL-1.0.html", + "reference": "https://spdx.org/licenses/MakeIndex.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CPAL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/MakeIndex.json", "referenceNumber": 79, - "name": "Common Public Attribution License 1.0", - "licenseId": "CPAL-1.0", + "name": "MakeIndex License", + "licenseId": "MakeIndex", "seeAlso": [ - "https://opensource.org/licenses/CPAL-1.0" + "https://fedoraproject.org/wiki/Licensing/MakeIndex" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Apache-1.1.html", + "reference": "https://spdx.org/licenses/LAL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Apache-1.1.json", + "detailsUrl": "https://spdx.org/licenses/LAL-1.2.json", "referenceNumber": 80, - "name": "Apache License 1.1", - "licenseId": "Apache-1.1", + "name": "Licence Art Libre 1.2", + "licenseId": "LAL-1.2", "seeAlso": [ - "http://apache.org/licenses/LICENSE-1.1", - "https://opensource.org/licenses/Apache-1.1" + "http://artlibre.org/licence/lal/licence-art-libre-12/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/UPL-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/UPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.json", "referenceNumber": 81, - "name": "Universal Permissive License v1.0", - "licenseId": "UPL-1.0", + "name": "GNU Free Documentation License v1.2 or later - no invariants", + "licenseId": "GFDL-1.2-no-invariants-or-later", "seeAlso": [ - "https://opensource.org/licenses/UPL" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-or-later.json", "referenceNumber": 82, - "name": "GNU Free Documentation License v1.1 or later - no invariants", - "licenseId": "GFDL-1.1-no-invariants-or-later", + "name": "GNU Free Documentation License v1.2 or later", + "licenseId": "GFDL-1.2-or-later", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.html", + "reference": "https://spdx.org/licenses/GLWTPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.json", + "detailsUrl": "https://spdx.org/licenses/GLWTPL.json", "referenceNumber": 83, - "name": "GNU Free Documentation License v1.2 or later - invariants", - "licenseId": "GFDL-1.2-invariants-or-later", + "name": "Good Luck With That Public License", + "licenseId": "GLWTPL", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://github.com/me-shaon/GLWTPL/commit/da5f6bc734095efbacb442c0b31e33a65b9d6e85" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/mpich2.html", + "reference": "https://spdx.org/licenses/OPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/mpich2.json", + "detailsUrl": "https://spdx.org/licenses/OPL-1.0.json", "referenceNumber": 84, - "name": "mpich2 License", - "licenseId": "mpich2", + "name": "Open Public License v1.0", + "licenseId": "OPL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT" + "http://old.koalateam.com/jackaroo/OPL_1_0.TXT", + "https://fedoraproject.org/wiki/Licensing/Open_Public_License" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.0-only.html", + "reference": "https://spdx.org/licenses/Unicode-DFS-2016.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-only.json", + "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2016.json", "referenceNumber": 85, - "name": "GNU Library General Public License v2 only", - "licenseId": "LGPL-2.0-only", + "name": "Unicode License Agreement - Data Files and Software (2016)", + "licenseId": "Unicode-DFS-2016", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + "http://www.unicode.org/copyright.html" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/eGenix.html", + "reference": "https://spdx.org/licenses/Parity-7.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/eGenix.json", + "detailsUrl": "https://spdx.org/licenses/Parity-7.0.0.json", "referenceNumber": 86, - "name": "eGenix.com Public License 1.1.0", - "licenseId": "eGenix", + "name": "The Parity Public License 7.0.0", + "licenseId": "Parity-7.0.0", "seeAlso": [ - "http://www.egenix.com/products/eGenix.com-Public-License-1.1.0.pdf", - "https://fedoraproject.org/wiki/Licensing/eGenix.com_Public_License_1.1.0" + "https://paritylicense.com/versions/7.0.0.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-only.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-LBNL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-only.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-LBNL.json", "referenceNumber": 87, - "name": "GNU Free Documentation License v1.2 only", - "licenseId": "GFDL-1.2-only", + "name": "Lawrence Berkeley National Labs BSD variant license", + "licenseId": "BSD-3-Clause-LBNL", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://fedoraproject.org/wiki/Licensing/LBNLBSD" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Unicode-TOU.html", + "reference": "https://spdx.org/licenses/Community-Spec-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unicode-TOU.json", + "detailsUrl": "https://spdx.org/licenses/Community-Spec-1.0.json", "referenceNumber": 88, - "name": "Unicode Terms of Use", - "licenseId": "Unicode-TOU", + "name": "Community Specification License 1.0", + "licenseId": "Community-Spec-1.0", "seeAlso": [ - "http://www.unicode.org/copyright.html" + "https://github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/APSL-1.2.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-1.2.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.json", "referenceNumber": 89, - "name": "Apple Public Source License 1.2", - "licenseId": "APSL-1.2", + "name": "GNU Free Documentation License v1.1 only - no invariants", + "licenseId": "GFDL-1.1-no-invariants-only", "seeAlso": [ - "http://www.samurajdata.se/opensource/mirror/licenses/apsl.php" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Apache-2.0.html", + "reference": "https://spdx.org/licenses/AGPL-1.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Apache-2.0.json", + "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-only.json", "referenceNumber": 90, - "name": "Apache License 2.0", - "licenseId": "Apache-2.0", + "name": "Affero General Public License v1.0 only", + "licenseId": "AGPL-1.0-only", "seeAlso": [ - "https://www.apache.org/licenses/LICENSE-2.0", - "https://opensource.org/licenses/Apache-2.0" + "http://www.affero.org/oagpl.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-Source-Code.html", + "reference": "https://spdx.org/licenses/RSCPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-Source-Code.json", + "detailsUrl": "https://spdx.org/licenses/RSCPL.json", "referenceNumber": 91, - "name": "BSD Source Code Attribution", - "licenseId": "BSD-Source-Code", + "name": "Ricoh Source Code Public License", + "licenseId": "RSCPL", "seeAlso": [ - "https://github.com/robbiehanson/CocoaHTTPServer/blob/master/LICENSE.txt" + "http://wayback.archive.org/web/20060715140826/http://www.risource.org/RPL/RPL-1.0A.shtml", + "https://opensource.org/licenses/RSCPL" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-4.0.html", + "reference": "https://spdx.org/licenses/SGI-B-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-4.0.json", + "detailsUrl": "https://spdx.org/licenses/SGI-B-1.1.json", "referenceNumber": 92, - "name": "Creative Commons Attribution Non Commercial 4.0 International", - "licenseId": "CC-BY-NC-4.0", + "name": "SGI Free Software License B v1.1", + "licenseId": "SGI-B-1.1", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/4.0/legalcode" + "http://oss.sgi.com/projects/FreeB/" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TMate.html", + "reference": "https://spdx.org/licenses/MIT-enna.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TMate.json", + "detailsUrl": "https://spdx.org/licenses/MIT-enna.json", "referenceNumber": 93, - "name": "TMate Open Source License", - "licenseId": "TMate", + "name": "enna License", + "licenseId": "MIT-enna", "seeAlso": [ - "http://svnkit.com/license.html" + "https://fedoraproject.org/wiki/Licensing/MIT#enna" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OPL-1.0.html", + "reference": "https://spdx.org/licenses/Sleepycat.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Sleepycat.json", "referenceNumber": 94, - "name": "Open Public License v1.0", - "licenseId": "OPL-1.0", + "name": "Sleepycat License", + "licenseId": "Sleepycat", "seeAlso": [ - "http://old.koalateam.com/jackaroo/OPL_1_0.TXT", - "https://fedoraproject.org/wiki/Licensing/Open_Public_License" + "https://opensource.org/licenses/Sleepycat" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CECILL-1.0.html", + "reference": "https://spdx.org/licenses/gSOAP-1.3b.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/gSOAP-1.3b.json", "referenceNumber": 95, - "name": "CeCILL Free Software License Agreement v1.0", - "licenseId": "CECILL-1.0", + "name": "gSOAP Public License v1.3b", + "licenseId": "gSOAP-1.3b", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V1-fr.html" + "http://www.cs.fsu.edu/~engelen/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EPL-2.0.html", + "reference": "https://spdx.org/licenses/CDLA-Sharing-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EPL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/CDLA-Sharing-1.0.json", "referenceNumber": 96, - "name": "Eclipse Public License 2.0", - "licenseId": "EPL-2.0", + "name": "Community Data License Agreement Sharing 1.0", + "licenseId": "CDLA-Sharing-1.0", "seeAlso": [ - "https://www.eclipse.org/legal/epl-2.0", - "https://www.opensource.org/licenses/EPL-2.0" + "https://cdla.io/sharing-1-0" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPPL-1.3c.html", + "reference": "https://spdx.org/licenses/Intel.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.3c.json", + "detailsUrl": "https://spdx.org/licenses/Intel.json", "referenceNumber": 97, - "name": "LaTeX Project Public License v1.3c", - "licenseId": "LPPL-1.3c", + "name": "Intel Open Source License", + "licenseId": "Intel", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-3c.txt", - "https://opensource.org/licenses/LPPL-1.3c" + "https://opensource.org/licenses/Intel" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Zed.html", + "reference": "https://spdx.org/licenses/CECILL-C.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zed.json", + "detailsUrl": "https://spdx.org/licenses/CECILL-C.json", "referenceNumber": 98, - "name": "Zed License", - "licenseId": "Zed", + "name": "CeCILL-C Free Software License Agreement", + "licenseId": "CECILL-C", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Zed" + "http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SGI-B-2.0.html", + "reference": "https://spdx.org/licenses/ZPL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SGI-B-2.0.json", + "detailsUrl": "https://spdx.org/licenses/ZPL-2.1.json", "referenceNumber": 99, - "name": "SGI Free Software License B v2.0", - "licenseId": "SGI-B-2.0", + "name": "Zope Public License 2.1", + "licenseId": "ZPL-2.1", "seeAlso": [ - "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf" + "http://old.zope.org/Resources/ZPL/" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.html", + "reference": "https://spdx.org/licenses/MIT-open-group.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.json", + "detailsUrl": "https://spdx.org/licenses/MIT-open-group.json", "referenceNumber": 100, - "name": "BSD 3-Clause Open MPI variant", - "licenseId": "BSD-3-Clause-Open-MPI", + "name": "MIT Open Group variant", + "licenseId": "MIT-open-group", "seeAlso": [ - "https://www.open-mpi.org/community/license.php", - "http://www.netlib.org/lapack/LICENSE.txt" + "https://gitlab.freedesktop.org/xorg/app/iceauth/-/blob/master/COPYING", + "https://gitlab.freedesktop.org/xorg/app/xvinfo/-/blob/master/COPYING", + "https://gitlab.freedesktop.org/xorg/app/xsetroot/-/blob/master/COPYING", + "https://gitlab.freedesktop.org/xorg/app/xauth/-/blob/master/COPYING" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDDL-1.0.html", + "reference": "https://spdx.org/licenses/Zimbra-1.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDDL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Zimbra-1.3.json", "referenceNumber": 101, - "name": "Common Development and Distribution License 1.0", - "licenseId": "CDDL-1.0", + "name": "Zimbra Public License v1.3", + "licenseId": "Zimbra-1.3", "seeAlso": [ - "https://opensource.org/licenses/cddl1" + "http://web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-or-later.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-or-later.json", + "reference": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.json", "referenceNumber": 102, - "name": "GNU Free Documentation License v1.2 or later", - "licenseId": "GFDL-1.2-or-later", + "name": "GNU General Public License v3.0 w/Autoconf exception", + "licenseId": "GPL-3.0-with-autoconf-exception", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://www.gnu.org/licenses/autoconf-exception-3.0.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Hippocratic-2.1.html", + "reference": "https://spdx.org/licenses/TU-Berlin-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Hippocratic-2.1.json", + "detailsUrl": "https://spdx.org/licenses/TU-Berlin-2.0.json", "referenceNumber": 103, - "name": "Hippocratic License 2.1", - "licenseId": "Hippocratic-2.1", + "name": "Technische Universitaet Berlin License 2.0", + "licenseId": "TU-Berlin-2.0", "seeAlso": [ - "https://firstdonoharm.dev/version/2/1/license.html", - "https://github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt" + "https://github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Linux-OpenIB.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Linux-OpenIB.json", + "reference": "https://spdx.org/licenses/wxWindows.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/wxWindows.json", "referenceNumber": 104, - "name": "Linux Kernel Variant of OpenIB.org license", - "licenseId": "Linux-OpenIB", + "name": "wxWindows Library License", + "licenseId": "wxWindows", "seeAlso": [ - "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h" + "https://opensource.org/licenses/WXwindows" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/IJG.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IJG.json", + "reference": "https://spdx.org/licenses/LGPL-2.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0.json", "referenceNumber": 105, - "name": "Independent JPEG Group License", - "licenseId": "IJG", + "name": "GNU Library General Public License v2 only", + "licenseId": "LGPL-2.0", "seeAlso": [ - "http://dev.w3.org/cvsweb/Amaya/libjpeg/Attic/README?rev\u003d1.2" + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/SchemeReport.html", + "reference": "https://spdx.org/licenses/PDDL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SchemeReport.json", + "detailsUrl": "https://spdx.org/licenses/PDDL-1.0.json", "referenceNumber": 106, - "name": "Scheme Language Report License", - "licenseId": "SchemeReport", - "seeAlso": [], + "name": "Open Data Commons Public Domain Dedication \u0026 License 1.0", + "licenseId": "PDDL-1.0", + "seeAlso": [ + "http://opendatacommons.org/licenses/pddl/1.0/", + "https://opendatacommons.org/licenses/pddl/" + ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TCL.html", + "reference": "https://spdx.org/licenses/Frameworx-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TCL.json", + "detailsUrl": "https://spdx.org/licenses/Frameworx-1.0.json", "referenceNumber": 107, - "name": "TCL/TK License", - "licenseId": "TCL", + "name": "Frameworx Open License 1.0", + "licenseId": "Frameworx-1.0", "seeAlso": [ - "http://www.tcl.tk/software/tcltk/license.html", - "https://fedoraproject.org/wiki/Licensing/TCL" + "https://opensource.org/licenses/Frameworx-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Linux-man-pages-copyleft.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Linux-man-pages-copyleft.json", + "reference": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.json", "referenceNumber": 108, - "name": "Linux man-pages Copyleft", - "licenseId": "Linux-man-pages-copyleft", + "name": "BSD 2-Clause FreeBSD License", + "licenseId": "BSD-2-Clause-FreeBSD", "seeAlso": [ - "https://www.kernel.org/doc/man-pages/licenses.html" + "http://www.freebsd.org/copyright/freebsd-license.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NBPL-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NBPL-1.0.json", + "reference": "https://spdx.org/licenses/AGPL-3.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/AGPL-3.0.json", "referenceNumber": 109, - "name": "Net Boolean Public License v1", - "licenseId": "NBPL-1.0", + "name": "GNU Affero General Public License v3.0", + "licenseId": "AGPL-3.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d37b4b3f6cc4bf34e1d3dec61e69914b9819d8894" + "https://www.gnu.org/licenses/agpl.txt", + "https://opensource.org/licenses/AGPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Zimbra-1.3.html", + "reference": "https://spdx.org/licenses/YPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zimbra-1.3.json", + "detailsUrl": "https://spdx.org/licenses/YPL-1.1.json", "referenceNumber": 110, - "name": "Zimbra Public License v1.3", - "licenseId": "Zimbra-1.3", + "name": "Yahoo! Public License v1.1", + "licenseId": "YPL-1.1", "seeAlso": [ - "http://web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html" + "http://www.zimbra.com/license/yahoo_public_license_1.1.html" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MPL-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5.json", "referenceNumber": 111, - "name": "Mozilla Public License 1.0", - "licenseId": "MPL-1.0", + "name": "Creative Commons Attribution 2.5 Generic", + "licenseId": "CC-BY-2.5", "seeAlso": [ - "http://www.mozilla.org/MPL/MPL-1.0.html", - "https://opensource.org/licenses/MPL-1.0" + "https://creativecommons.org/licenses/by/2.5/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.1-or-later.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-or-later.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-4.0.json", "referenceNumber": 112, - "name": "GNU Lesser General Public License v2.1 or later", - "licenseId": "LGPL-2.1-or-later", + "name": "Creative Commons Attribution Share Alike 4.0 International", + "licenseId": "CC-BY-SA-4.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", - "https://opensource.org/licenses/LGPL-2.1" + "https://creativecommons.org/licenses/by-sa/4.0/legalcode" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-PDDC.html", + "reference": "https://spdx.org/licenses/Adobe-2006.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-PDDC.json", + "detailsUrl": "https://spdx.org/licenses/Adobe-2006.json", "referenceNumber": 113, - "name": "Creative Commons Public Domain Dedication and Certification", - "licenseId": "CC-PDDC", + "name": "Adobe Systems Incorporated Source Code License Agreement", + "licenseId": "Adobe-2006", "seeAlso": [ - "https://creativecommons.org/licenses/publicdomain/" + "https://fedoraproject.org/wiki/Licensing/AdobeLicense" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Imlib2.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Imlib2.json", + "reference": "https://spdx.org/licenses/LGPL-2.1+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1+.json", "referenceNumber": 114, - "name": "Imlib2 License", - "licenseId": "Imlib2", + "name": "GNU Library General Public License v2.1 or later", + "licenseId": "LGPL-2.1+", "seeAlso": [ - "http://trac.enlightenment.org/e/browser/trunk/imlib2/COPYING", - "https://git.enlightenment.org/legacy/imlib2.git/tree/COPYING" + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Multics.html", + "reference": "https://spdx.org/licenses/OGC-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Multics.json", + "detailsUrl": "https://spdx.org/licenses/OGC-1.0.json", "referenceNumber": 115, - "name": "Multics License", - "licenseId": "Multics", + "name": "OGC Software License, Version 1.0", + "licenseId": "OGC-1.0", "seeAlso": [ - "https://opensource.org/licenses/Multics" + "https://www.ogc.org/ogc/software/1.0" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/D-FSL-1.0.html", + "reference": "https://spdx.org/licenses/AGPL-3.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/D-FSL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-only.json", "referenceNumber": 116, - "name": "Deutsche Freie Software Lizenz", - "licenseId": "D-FSL-1.0", + "name": "GNU Affero General Public License v3.0 only", + "licenseId": "AGPL-3.0-only", "seeAlso": [ - "http://www.dipp.nrw.de/d-fsl/lizenzen/", - "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/de/D-FSL-1_0_de.txt", - "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/en/D-FSL-1_0_en.txt", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/deutsche-freie-software-lizenz", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/german-free-software-license", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_de.txt/at_download/file", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_en.txt/at_download/file" + "https://www.gnu.org/licenses/agpl.txt", + "https://opensource.org/licenses/AGPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.html", + "reference": "https://spdx.org/licenses/Vim.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.json", + "detailsUrl": "https://spdx.org/licenses/Vim.json", "referenceNumber": 117, - "name": "BSD 3-Clause No Nuclear Warranty", - "licenseId": "BSD-3-Clause-No-Nuclear-Warranty", + "name": "Vim License", + "licenseId": "Vim", "seeAlso": [ - "https://jogamp.org/git/?p\u003dgluegen.git;a\u003dblob_plain;f\u003dLICENSE.txt" + "http://vimdoc.sourceforge.net/htmldoc/uganda.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BlueOak-1.0.0.html", + "reference": "https://spdx.org/licenses/ECL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BlueOak-1.0.0.json", + "detailsUrl": "https://spdx.org/licenses/ECL-2.0.json", "referenceNumber": 118, - "name": "Blue Oak Model License 1.0.0", - "licenseId": "BlueOak-1.0.0", + "name": "Educational Community License v2.0", + "licenseId": "ECL-2.0", "seeAlso": [ - "https://blueoakcouncil.org/license/1.0.0" + "https://opensource.org/licenses/ECL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Community-Spec-1.0.html", + "reference": "https://spdx.org/licenses/UCL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Community-Spec-1.0.json", + "detailsUrl": "https://spdx.org/licenses/UCL-1.0.json", "referenceNumber": 119, - "name": "Community Specification License 1.0", - "licenseId": "Community-Spec-1.0", + "name": "Upstream Compatibility License v1.0", + "licenseId": "UCL-1.0", "seeAlso": [ - "https://github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md" + "https://opensource.org/licenses/UCL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-only.html", + "reference": "https://spdx.org/licenses/mplus.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-only.json", + "detailsUrl": "https://spdx.org/licenses/mplus.json", "referenceNumber": 120, - "name": "GNU Free Documentation License v1.1 only", - "licenseId": "GFDL-1.1-only", + "name": "mplus Font License", + "licenseId": "mplus", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://fedoraproject.org/wiki/Licensing:Mplus?rd\u003dLicensing/mplus" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/RPL-1.5.html", + "reference": "https://spdx.org/licenses/BlueOak-1.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RPL-1.5.json", + "detailsUrl": "https://spdx.org/licenses/BlueOak-1.0.0.json", "referenceNumber": 121, - "name": "Reciprocal Public License 1.5", - "licenseId": "RPL-1.5", + "name": "Blue Oak Model License 1.0.0", + "licenseId": "BlueOak-1.0.0", "seeAlso": [ - "https://opensource.org/licenses/RPL-1.5" + "https://blueoakcouncil.org/license/1.0.0" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OFL-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.0.json", "referenceNumber": 122, - "name": "SIL Open Font License 1.1", - "licenseId": "OFL-1.1", + "name": "Creative Commons Attribution Share Alike 2.0 Generic", + "licenseId": "CC-BY-SA-2.0", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", - "https://opensource.org/licenses/OFL-1.1" + "https://creativecommons.org/licenses/by-sa/2.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CPL-1.0.html", + "reference": "https://spdx.org/licenses/AFL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/AFL-2.1.json", "referenceNumber": 123, - "name": "Common Public License 1.0", - "licenseId": "CPL-1.0", + "name": "Academic Free License v2.1", + "licenseId": "AFL-2.1", "seeAlso": [ - "https://opensource.org/licenses/CPL-1.0" + "http://opensource.linux-mirror.org/licenses/afl-2.1.txt" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/AGPL-3.0-only.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-only.json", - "referenceNumber": 124, - "name": "GNU Affero General Public License v3.0 only", - "licenseId": "AGPL-3.0-only", + "reference": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.json", + "referenceNumber": 124, + "name": "GNU General Public License v2.0 w/Classpath exception", + "licenseId": "GPL-2.0-with-classpath-exception", "seeAlso": [ - "https://www.gnu.org/licenses/agpl.txt", - "https://opensource.org/licenses/AGPL-3.0" + "https://www.gnu.org/software/classpath/license.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/gnuplot.html", + "reference": "https://spdx.org/licenses/OLDAP-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/gnuplot.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.2.json", "referenceNumber": 125, - "name": "gnuplot License", - "licenseId": "gnuplot", + "name": "Open LDAP Public License v1.2", + "licenseId": "OLDAP-1.2", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Gnuplot" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d42b0383c50c299977b5893ee695cf4e486fb0dc7" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NLPL.html", + "reference": "https://spdx.org/licenses/NGPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NLPL.json", + "detailsUrl": "https://spdx.org/licenses/NGPL.json", "referenceNumber": 126, - "name": "No Limit Public License", - "licenseId": "NLPL", + "name": "Nethack General Public License", + "licenseId": "NGPL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/NLPL" + "https://opensource.org/licenses/NGPL" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/ADSL.html", + "reference": "https://spdx.org/licenses/App-s2p.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ADSL.json", + "detailsUrl": "https://spdx.org/licenses/App-s2p.json", "referenceNumber": 127, - "name": "Amazon Digital Services License", - "licenseId": "ADSL", + "name": "App::s2p License", + "licenseId": "App-s2p", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AmazonDigitalServicesLicense" + "https://fedoraproject.org/wiki/Licensing/App-s2p" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/psfrag.html", + "reference": "https://spdx.org/licenses/DL-DE-BY-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/psfrag.json", + "detailsUrl": "https://spdx.org/licenses/DL-DE-BY-2.0.json", "referenceNumber": 128, - "name": "psfrag License", - "licenseId": "psfrag", + "name": "Data licence Germany – attribution – version 2.0", + "licenseId": "DL-DE-BY-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/psfrag" + "https://www.govdata.de/dl-de/by-2-0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Watcom-1.0.html", + "reference": "https://spdx.org/licenses/OGL-Canada-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Watcom-1.0.json", + "detailsUrl": "https://spdx.org/licenses/OGL-Canada-2.0.json", "referenceNumber": 129, - "name": "Sybase Open Watcom Public License 1.0", - "licenseId": "Watcom-1.0", + "name": "Open Government Licence - Canada", + "licenseId": "OGL-Canada-2.0", "seeAlso": [ - "https://opensource.org/licenses/Watcom-1.0" + "https://open.canada.ca/en/open-government-licence-canada" ], - "isOsiApproved": true, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SGI-B-1.0.html", + "reference": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SGI-B-1.0.json", + "detailsUrl": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.json", "referenceNumber": 130, - "name": "SGI Free Software License B v1.0", - "licenseId": "SGI-B-1.0", + "name": "Licence Libre du Québec – Réciprocité forte version 1.1", + "licenseId": "LiLiQ-Rplus-1.1", "seeAlso": [ - "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html" + "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-forte-liliq-r-v1-1/", + "http://opensource.org/licenses/LiLiQ-Rplus-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/W3C-19980720.html", + "reference": "https://spdx.org/licenses/bzip2-1.0.6.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/W3C-19980720.json", + "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.6.json", "referenceNumber": 131, - "name": "W3C Software Notice and License (1998-07-20)", - "licenseId": "W3C-19980720", + "name": "bzip2 and libbzip2 License v1.0.6", + "licenseId": "bzip2-1.0.6", "seeAlso": [ - "http://www.w3.org/Consortium/Legal/copyright-software-19980720.html" + "https://sourceware.org/git/?p\u003dbzip2.git;a\u003dblob;f\u003dLICENSE;hb\u003dbzip2-1.0.6", + "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NASA-1.3.html", + "reference": "https://spdx.org/licenses/IBM-pibs.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NASA-1.3.json", + "detailsUrl": "https://spdx.org/licenses/IBM-pibs.json", "referenceNumber": 132, - "name": "NASA Open Source Agreement 1.3", - "licenseId": "NASA-1.3", + "name": "IBM PowerPC Initialization and Boot Software", + "licenseId": "IBM-pibs", "seeAlso": [ - "http://ti.arc.nasa.gov/opensource/nosa/", - "https://opensource.org/licenses/NASA-1.3" + "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003darch/powerpc/cpu/ppc4xx/miiphy.c;h\u003d297155fdafa064b955e53e9832de93bfb0cfb85b;hb\u003d9fab4bf4cc077c21e43941866f3f2c196f28670d" ], - "isOsiApproved": true, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Fair.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Fair.json", + "reference": "https://spdx.org/licenses/GFDL-1.3.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3.json", "referenceNumber": 133, - "name": "Fair License", - "licenseId": "Fair", + "name": "GNU Free Documentation License v1.3", + "licenseId": "GFDL-1.3", "seeAlso": [ - "http://fairlicense.org/", - "https://opensource.org/licenses/Fair" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CERN-OHL-1.1.html", + "reference": "https://spdx.org/licenses/OLDAP-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.1.json", "referenceNumber": 134, - "name": "CERN Open Hardware Licence v1.1", - "licenseId": "CERN-OHL-1.1", + "name": "Open LDAP Public License v1.1", + "licenseId": "OLDAP-1.1", "seeAlso": [ - "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d806557a5ad59804ef3a44d5abfbe91d706b0791f" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-3.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0+.json", + "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-only.json", "referenceNumber": 135, - "name": "GNU Lesser General Public License v3.0 or later", - "licenseId": "LGPL-3.0+", + "name": "GNU Free Documentation License v1.1 only - invariants", + "licenseId": "GFDL-1.1-invariants-only", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://opensource.org/licenses/LGPL-3.0" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-3.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0.json", + "reference": "https://spdx.org/licenses/OLDAP-2.2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.json", "referenceNumber": 136, - "name": "GNU Lesser General Public License v3.0 only", - "licenseId": "LGPL-3.0", + "name": "Open LDAP Public License v2.2", + "licenseId": "OLDAP-2.2", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://opensource.org/licenses/LGPL-3.0" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d470b0c18ec67621c85881b2733057fecf4a1acc3" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-Views.html", + "reference": "https://spdx.org/licenses/NIST-PD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Views.json", + "detailsUrl": "https://spdx.org/licenses/NIST-PD.json", "referenceNumber": 137, - "name": "BSD 2-Clause with views sentence", - "licenseId": "BSD-2-Clause-Views", + "name": "NIST Public Domain Notice", + "licenseId": "NIST-PD", "seeAlso": [ - "http://www.freebsd.org/copyright/freebsd-license.html", - "https://people.freebsd.org/~ivoras/wine/patch-wine-nvidia.sh", - "https://github.com/protegeproject/protege/blob/master/license.txt" + "https://github.com/tcheneau/simpleRPL/blob/e645e69e38dd4e3ccfeceb2db8cba05b7c2e0cd3/LICENSE.txt", + "https://github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-1.1.html", + "reference": "https://spdx.org/licenses/EUPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.1.json", + "detailsUrl": "https://spdx.org/licenses/EUPL-1.0.json", "referenceNumber": 138, - "name": "Open LDAP Public License v1.1", - "licenseId": "OLDAP-1.1", + "name": "European Union Public License 1.0", + "licenseId": "EUPL-1.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d806557a5ad59804ef3a44d5abfbe91d706b0791f" + "http://ec.europa.eu/idabc/en/document/7330.html", + "http://ec.europa.eu/idabc/servlets/Doc027f.pdf?id\u003d31096" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.html", + "reference": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json", + "detailsUrl": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.json", "referenceNumber": 139, - "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", - "licenseId": "CC-BY-NC-ND-3.0-DE", + "name": "Mozilla Public License 2.0 (no copyleft exception)", + "licenseId": "MPL-2.0-no-copyleft-exception", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/3.0/de/legalcode" + "https://www.mozilla.org/MPL/2.0/", + "https://opensource.org/licenses/MPL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OGDL-Taiwan-1.0.html", + "reference": "https://spdx.org/licenses/CC0-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGDL-Taiwan-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CC0-1.0.json", "referenceNumber": 140, - "name": "Taiwan Open Government Data License, version 1.0", - "licenseId": "OGDL-Taiwan-1.0", + "name": "Creative Commons Zero v1.0 Universal", + "licenseId": "CC0-1.0", "seeAlso": [ - "https://data.gov.tw/license" + "https://creativecommons.org/publicdomain/zero/1.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NGPL.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-US.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NGPL.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-US.json", "referenceNumber": 141, - "name": "Nethack General Public License", - "licenseId": "NGPL", + "name": "Creative Commons Attribution 3.0 United States", + "licenseId": "CC-BY-3.0-US", "seeAlso": [ - "https://opensource.org/licenses/NGPL" + "https://creativecommons.org/licenses/by/3.0/us/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSL-1.0.html", + "reference": "https://spdx.org/licenses/EUPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/EUPL-1.1.json", "referenceNumber": 142, - "name": "Boost Software License 1.0", - "licenseId": "BSL-1.0", + "name": "European Union Public License 1.1", + "licenseId": "EUPL-1.1", "seeAlso": [ - "http://www.boost.org/LICENSE_1_0.txt", - "https://opensource.org/licenses/BSL-1.0" + "https://joinup.ec.europa.eu/software/page/eupl/licence-eupl", + "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl1.1.-licence-en_0.pdf", + "https://opensource.org/licenses/EUPL-1.1" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Sendmail.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Sendmail.json", + "reference": "https://spdx.org/licenses/GPL-3.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0+.json", "referenceNumber": 143, - "name": "Sendmail License", - "licenseId": "Sendmail", + "name": "GNU General Public License v3.0 or later", + "licenseId": "GPL-3.0+", "seeAlso": [ - "http://www.sendmail.com/pdfs/open_source/sendmail_license.pdf", - "https://web.archive.org/web/20160322142305/https://www.sendmail.com/pdfs/open_source/sendmail_license.pdf" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.4.html", + "reference": "https://spdx.org/licenses/OLDAP-2.8.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.4.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.8.json", "referenceNumber": 144, - "name": "Open LDAP Public License v2.4", - "licenseId": "OLDAP-2.4", + "name": "Open LDAP Public License v2.8", + "licenseId": "OLDAP-2.8", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcd1284c4a91a8a380d904eee68d1583f989ed386" + "http://www.openldap.org/software/release/license.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CrystalStacker.html", + "reference": "https://spdx.org/licenses/OSL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CrystalStacker.json", + "detailsUrl": "https://spdx.org/licenses/OSL-2.1.json", "referenceNumber": 145, - "name": "CrystalStacker License", - "licenseId": "CrystalStacker", + "name": "Open Software License 2.1", + "licenseId": "OSL-2.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:CrystalStacker?rd\u003dLicensing/CrystalStacker" + "http://web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm", + "https://opensource.org/licenses/OSL-2.1" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/blessing.html", + "reference": "https://spdx.org/licenses/libpng-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/blessing.json", + "detailsUrl": "https://spdx.org/licenses/libpng-2.0.json", "referenceNumber": 146, - "name": "SQLite Blessing", - "licenseId": "blessing", + "name": "PNG Reference Library version 2", + "licenseId": "libpng-2.0", "seeAlso": [ - "https://www.sqlite.org/src/artifact/e33a4df7e32d742a?ln\u003d4-9", - "https://sqlite.org/src/artifact/df5091916dbb40e6" + "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-4.0.html", + "reference": "https://spdx.org/licenses/O-UDA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-4.0.json", + "detailsUrl": "https://spdx.org/licenses/O-UDA-1.0.json", "referenceNumber": 147, - "name": "Creative Commons Attribution Share Alike 4.0 International", - "licenseId": "CC-BY-SA-4.0", + "name": "Open Use of Data Agreement v1.0", + "licenseId": "O-UDA-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/4.0/legalcode" + "https://github.com/microsoft/Open-Use-of-Data-Agreement/blob/v1.0/O-UDA-1.0.md", + "https://cdla.dev/open-use-of-data-agreement-v1-0/" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-2.5.html", + "reference": "https://spdx.org/licenses/gnuplot.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5.json", + "detailsUrl": "https://spdx.org/licenses/gnuplot.json", "referenceNumber": 148, - "name": "Creative Commons Attribution 2.5 Generic", - "licenseId": "CC-BY-2.5", + "name": "gnuplot License", + "licenseId": "gnuplot", "seeAlso": [ - "https://creativecommons.org/licenses/by/2.5/legalcode" + "https://fedoraproject.org/wiki/Licensing/Gnuplot" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/ICU.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Modification.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ICU.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Modification.json", "referenceNumber": 149, - "name": "ICU License", - "licenseId": "ICU", + "name": "BSD 3-Clause Modification", + "licenseId": "BSD-3-Clause-Modification", "seeAlso": [ - "http://source.icu-project.org/repos/icu/icu/trunk/license.html" + "https://fedoraproject.org/wiki/Licensing:BSD#Modification_Variant" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/diffmark.html", + "reference": "https://spdx.org/licenses/ODC-By-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/diffmark.json", + "detailsUrl": "https://spdx.org/licenses/ODC-By-1.0.json", "referenceNumber": 150, - "name": "diffmark license", - "licenseId": "diffmark", + "name": "Open Data Commons Attribution License v1.0", + "licenseId": "ODC-By-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/diffmark" + "https://opendatacommons.org/licenses/by/1.0/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/libpng-2.0.html", + "reference": "https://spdx.org/licenses/Imlib2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/libpng-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Imlib2.json", "referenceNumber": 151, - "name": "PNG Reference Library version 2", - "licenseId": "libpng-2.0", + "name": "Imlib2 License", + "licenseId": "Imlib2", "seeAlso": [ - "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" + "http://trac.enlightenment.org/e/browser/trunk/imlib2/COPYING", + "https://git.enlightenment.org/legacy/imlib2.git/tree/COPYING" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-1.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0+.json", + "reference": "https://spdx.org/licenses/OpenSSL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OpenSSL.json", "referenceNumber": 152, - "name": "GNU General Public License v1.0 or later", - "licenseId": "GPL-1.0+", + "name": "OpenSSL License", + "licenseId": "OpenSSL", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + "http://www.openssl.org/source/license.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/RSCPL.html", + "reference": "https://spdx.org/licenses/BUSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RSCPL.json", + "detailsUrl": "https://spdx.org/licenses/BUSL-1.1.json", "referenceNumber": 153, - "name": "Ricoh Source Code Public License", - "licenseId": "RSCPL", + "name": "Business Source License 1.1", + "licenseId": "BUSL-1.1", "seeAlso": [ - "http://wayback.archive.org/web/20060715140826/http://www.risource.org/RPL/RPL-1.0A.shtml", - "https://opensource.org/licenses/RSCPL" + "https://mariadb.com/bsl11/" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FSFAP.html", + "reference": "https://spdx.org/licenses/EUDatagrid.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FSFAP.json", + "detailsUrl": "https://spdx.org/licenses/EUDatagrid.json", "referenceNumber": 154, - "name": "FSF All Permissive License", - "licenseId": "FSFAP", + "name": "EU DataGrid Software License", + "licenseId": "EUDatagrid", "seeAlso": [ - "https://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html" + "http://eu-datagrid.web.cern.ch/eu-datagrid/license.html", + "https://opensource.org/licenses/EUDatagrid" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NPL-1.1.html", + "reference": "https://spdx.org/licenses/EFL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/EFL-2.0.json", "referenceNumber": 155, - "name": "Netscape Public License v1.1", - "licenseId": "NPL-1.1", + "name": "Eiffel Forum License v2.0", + "licenseId": "EFL-2.0", "seeAlso": [ - "http://www.mozilla.org/MPL/NPL/1.1/" + "http://www.eiffel-nice.org/license/eiffel-forum-license-2.html", + "https://opensource.org/licenses/EFL-2.0" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OSL-1.1.html", + "reference": "https://spdx.org/licenses/NRL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/NRL.json", "referenceNumber": 156, - "name": "Open Software License 1.1", - "licenseId": "OSL-1.1", + "name": "NRL License", + "licenseId": "NRL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/OSL1.1" + "http://web.mit.edu/network/isakmp/nrllicense.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MIT-feh.html", + "reference": "https://spdx.org/licenses/OSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-feh.json", + "detailsUrl": "https://spdx.org/licenses/OSL-1.1.json", "referenceNumber": 157, - "name": "feh License", - "licenseId": "MIT-feh", + "name": "Open Software License 1.1", + "licenseId": "OSL-1.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT#feh" + "https://fedoraproject.org/wiki/Licensing/OSL1.1" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/PDDL-1.0.html", + "reference": "https://spdx.org/licenses/RHeCos-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PDDL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/RHeCos-1.1.json", "referenceNumber": 158, - "name": "Open Data Commons Public Domain Dedication \u0026 License 1.0", - "licenseId": "PDDL-1.0", + "name": "Red Hat eCos Public License v1.1", + "licenseId": "RHeCos-1.1", "seeAlso": [ - "http://opendatacommons.org/licenses/pddl/1.0/", - "https://opendatacommons.org/licenses/pddl/" + "http://ecos.sourceware.org/old-license.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CERN-OHL-1.2.html", + "reference": "https://spdx.org/licenses/JasPer-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.2.json", + "detailsUrl": "https://spdx.org/licenses/JasPer-2.0.json", "referenceNumber": 159, - "name": "CERN Open Hardware Licence v1.2", - "licenseId": "CERN-OHL-1.2", + "name": "JasPer License", + "licenseId": "JasPer-2.0", "seeAlso": [ - "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2" + "http://www.ece.uvic.ca/~mdadams/jasper/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.json", "referenceNumber": 160, - "name": "Creative Commons Attribution 3.0 Unported", - "licenseId": "CC-BY-3.0", + "name": "GNU Free Documentation License v1.2 only - no invariants", + "licenseId": "GFDL-1.2-no-invariants-only", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/legalcode" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json", "referenceNumber": 161, - "name": "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", - "licenseId": "CC-BY-NC-ND-2.0", + "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + "licenseId": "CC-BY-NC-SA-2.0-UK", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/2.0/legalcode" + "https://creativecommons.org/licenses/by-nc-sa/2.0/uk/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-or-later.html", + "reference": "https://spdx.org/licenses/MulanPSL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-or-later.json", + "detailsUrl": "https://spdx.org/licenses/MulanPSL-2.0.json", "referenceNumber": 162, - "name": "GNU Free Documentation License v1.1 or later", - "licenseId": "GFDL-1.1-or-later", + "name": "Mulan Permissive Software License, Version 2", + "licenseId": "MulanPSL-2.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://license.coscl.org.cn/MulanPSL2/" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Borceux.html", + "reference": "https://spdx.org/licenses/XFree86-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Borceux.json", + "detailsUrl": "https://spdx.org/licenses/XFree86-1.1.json", "referenceNumber": 163, - "name": "Borceux license", - "licenseId": "Borceux", + "name": "XFree86 License 1.1", + "licenseId": "XFree86-1.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Borceux" + "http://www.xfree86.org/current/LICENSE4.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NCSA.html", + "reference": "https://spdx.org/licenses/SMPPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NCSA.json", + "detailsUrl": "https://spdx.org/licenses/SMPPL.json", "referenceNumber": 164, - "name": "University of Illinois/NCSA Open Source License", - "licenseId": "NCSA", + "name": "Secure Messaging Protocol Public License", + "licenseId": "SMPPL", "seeAlso": [ - "http://otm.illinois.edu/uiuc_openSource", - "https://opensource.org/licenses/NCSA" + "https://github.com/dcblake/SMP/blob/master/Documentation/License.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Naumen.html", + "reference": "https://spdx.org/licenses/CDDL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Naumen.json", + "detailsUrl": "https://spdx.org/licenses/CDDL-1.1.json", "referenceNumber": 165, - "name": "Naumen Public License", - "licenseId": "Naumen", + "name": "Common Development and Distribution License 1.1", + "licenseId": "CDDL-1.1", "seeAlso": [ - "https://opensource.org/licenses/Naumen" + "http://glassfish.java.net/public/CDDL+GPL_1_1.html", + "https://javaee.github.io/glassfish/LICENSE" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Caldera.html", + "reference": "https://spdx.org/licenses/JPNIC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Caldera.json", + "detailsUrl": "https://spdx.org/licenses/JPNIC.json", "referenceNumber": 166, - "name": "Caldera License", - "licenseId": "Caldera", + "name": "Japan Network Information Center License", + "licenseId": "JPNIC", "seeAlso": [ - "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf" + "https://gitlab.isc.org/isc-projects/bind9/blob/master/COPYRIGHT#L366" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/PHP-3.01.html", + "reference": "https://spdx.org/licenses/CDLA-Permissive-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PHP-3.01.json", + "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-1.0.json", "referenceNumber": 167, - "name": "PHP License v3.01", - "licenseId": "PHP-3.01", + "name": "Community Data License Agreement Permissive 1.0", + "licenseId": "CDLA-Permissive-1.0", "seeAlso": [ - "http://www.php.net/license/3_01.txt" + "https://cdla.io/permissive-1-0" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0.json", + "reference": "https://spdx.org/licenses/xinetd.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/xinetd.json", "referenceNumber": 168, - "name": "GNU General Public License v3.0 only", - "licenseId": "GPL-3.0", + "name": "xinetd License", + "licenseId": "xinetd", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "https://fedoraproject.org/wiki/Licensing/Xinetd_License" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.html", + "reference": "https://spdx.org/licenses/X11-distribute-modifications-variant.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.json", + "detailsUrl": "https://spdx.org/licenses/X11-distribute-modifications-variant.json", "referenceNumber": 169, - "name": "GNU Free Documentation License v1.1 only - no invariants", - "licenseId": "GFDL-1.1-no-invariants-only", + "name": "X11 License Distribution Modification Variant", + "licenseId": "X11-distribute-modifications-variant", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://github.com/mirror/ncurses/blob/master/COPYING" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/DOC.html", + "reference": "https://spdx.org/licenses/OLDAP-2.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/DOC.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.3.json", "referenceNumber": 170, - "name": "DOC License", - "licenseId": "DOC", + "name": "Open LDAP Public License v2.3", + "licenseId": "OLDAP-2.3", "seeAlso": [ - "http://www.cs.wustl.edu/~schmidt/ACE-copying.html", - "https://www.dre.vanderbilt.edu/~schmidt/ACE-copying.html" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dd32cf54a32d581ab475d23c810b0a7fbaf8d63c3" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OGC-1.0.html", + "reference": "https://spdx.org/licenses/SCEA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGC-1.0.json", + "detailsUrl": "https://spdx.org/licenses/SCEA.json", "referenceNumber": 171, - "name": "OGC Software License, Version 1.0", - "licenseId": "OGC-1.0", + "name": "SCEA Shared Source License", + "licenseId": "SCEA", "seeAlso": [ - "https://www.ogc.org/ogc/software/1.0" + "http://research.scea.com/scea_shared_source_license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0-or-later.html", + "reference": "https://spdx.org/licenses/SugarCRM-1.1.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-or-later.json", + "detailsUrl": "https://spdx.org/licenses/SugarCRM-1.1.3.json", "referenceNumber": 172, - "name": "GNU General Public License v3.0 or later", - "licenseId": "GPL-3.0-or-later", + "name": "SugarCRM Public License v1.1.3", + "licenseId": "SugarCRM-1.1.3", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "http://www.sugarcrm.com/crm/SPL" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SAX-PD.html", + "reference": "https://spdx.org/licenses/LGPL-2.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SAX-PD.json", + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-only.json", "referenceNumber": 173, - "name": "Sax Public Domain Notice", - "licenseId": "SAX-PD", + "name": "GNU Library General Public License v2 only", + "licenseId": "LGPL-2.0-only", "seeAlso": [ - "http://www.saxproject.org/copying.html" + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CNRI-Python.html", + "reference": "https://spdx.org/licenses/CDDL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CNRI-Python.json", + "detailsUrl": "https://spdx.org/licenses/CDDL-1.0.json", "referenceNumber": 174, - "name": "CNRI Python License", - "licenseId": "CNRI-Python", + "name": "Common Development and Distribution License 1.0", + "licenseId": "CDDL-1.0", "seeAlso": [ - "https://opensource.org/licenses/CNRI-Python" + "https://opensource.org/licenses/cddl1" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/HPND.html", + "reference": "https://spdx.org/licenses/SGI-B-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/HPND.json", + "detailsUrl": "https://spdx.org/licenses/SGI-B-2.0.json", "referenceNumber": 175, - "name": "Historical Permission Notice and Disclaimer", - "licenseId": "HPND", + "name": "SGI Free Software License B v2.0", + "licenseId": "SGI-B-2.0", "seeAlso": [ - "https://opensource.org/licenses/HPND" + "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CECILL-C.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-C.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.json", "referenceNumber": 176, - "name": "CeCILL-C Free Software License Agreement", - "licenseId": "CECILL-C", + "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + "licenseId": "CC-BY-NC-SA-3.0", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html" + "https://creativecommons.org/licenses/by-nc-sa/3.0/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-only.html", + "reference": "https://spdx.org/licenses/NetCDF.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-only.json", + "detailsUrl": "https://spdx.org/licenses/NetCDF.json", "referenceNumber": 177, - "name": "GNU General Public License v2.0 only", - "licenseId": "GPL-2.0-only", + "name": "NetCDF license", + "licenseId": "NetCDF", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "http://www.unidata.ucar.edu/software/netcdf/copyright.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/PSF-2.0.html", + "reference": "https://spdx.org/licenses/HaskellReport.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PSF-2.0.json", + "detailsUrl": "https://spdx.org/licenses/HaskellReport.json", "referenceNumber": 178, - "name": "Python Software Foundation License 2.0", - "licenseId": "PSF-2.0", + "name": "Haskell Language Report License", + "licenseId": "HaskellReport", "seeAlso": [ - "https://opensource.org/licenses/Python-2.0" + "https://fedoraproject.org/wiki/Licensing/Haskell_Language_Report_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OFL-1.1-no-RFN.html", + "reference": "https://spdx.org/licenses/LGPL-2.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.1-no-RFN.json", + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-or-later.json", "referenceNumber": 179, - "name": "SIL Open Font License 1.1 with no Reserved Font Name", - "licenseId": "OFL-1.1-no-RFN", + "name": "GNU Library General Public License v2 or later", + "licenseId": "LGPL-2.0-or-later", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", - "https://opensource.org/licenses/OFL-1.1" + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/W3C.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/W3C.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.json", "referenceNumber": 180, - "name": "W3C Software Notice and License (2002-12-31)", - "licenseId": "W3C", + "name": "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + "licenseId": "CC-BY-NC-ND-4.0", "seeAlso": [ - "http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231.html", - "https://opensource.org/licenses/W3C" + "https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OFL-1.0-no-RFN.html", + "reference": "https://spdx.org/licenses/BSD-1-Clause.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.0-no-RFN.json", + "detailsUrl": "https://spdx.org/licenses/BSD-1-Clause.json", "referenceNumber": 181, - "name": "SIL Open Font License 1.0 with no Reserved Font Name", - "licenseId": "OFL-1.0-no-RFN", + "name": "BSD 1-Clause License", + "licenseId": "BSD-1-Clause", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" + "https://svnweb.freebsd.org/base/head/include/ifaddrs.h?revision\u003d326823" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/xinetd.html", + "reference": "https://spdx.org/licenses/COIL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/xinetd.json", + "detailsUrl": "https://spdx.org/licenses/COIL-1.0.json", "referenceNumber": 182, - "name": "xinetd License", - "licenseId": "xinetd", + "name": "Copyfree Open Innovation License", + "licenseId": "COIL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Xinetd_License" + "https://coil.apotheon.org/plaintext/01.0.txt" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGTSL.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGTSL.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-4.0.json", "referenceNumber": 183, - "name": "Open Group Test Suite License", - "licenseId": "OGTSL", + "name": "Creative Commons Attribution No Derivatives 4.0 International", + "licenseId": "CC-BY-ND-4.0", "seeAlso": [ - "http://www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt", - "https://opensource.org/licenses/OGTSL" + "https://creativecommons.org/licenses/by-nd/4.0/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.3.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3.json", + "reference": "https://spdx.org/licenses/ANTLR-PD.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ANTLR-PD.json", "referenceNumber": 184, - "name": "GNU Free Documentation License v1.3", - "licenseId": "GFDL-1.3", + "name": "ANTLR Software Rights Notice", + "licenseId": "ANTLR-PD", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "http://www.antlr2.org/license.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AGPL-1.0-only.html", + "reference": "https://spdx.org/licenses/BSD-4-Clause-Shortened.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-only.json", + "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-Shortened.json", "referenceNumber": 185, - "name": "Affero General Public License v1.0 only", - "licenseId": "AGPL-1.0-only", + "name": "BSD 4 Clause Shortened", + "licenseId": "BSD-4-Clause-Shortened", "seeAlso": [ - "http://www.affero.org/oagpl.html" + "https://metadata.ftp-master.debian.org/changelogs//main/a/arpwatch/arpwatch_2.1a15-7_copyright" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/WTFPL.html", + "reference": "https://spdx.org/licenses/diffmark.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/WTFPL.json", + "detailsUrl": "https://spdx.org/licenses/diffmark.json", "referenceNumber": 186, - "name": "Do What The F*ck You Want To Public License", - "licenseId": "WTFPL", + "name": "diffmark license", + "licenseId": "diffmark", "seeAlso": [ - "http://www.wtfpl.net/about/", - "http://sam.zoy.org/wtfpl/COPYING" + "https://fedoraproject.org/wiki/Licensing/diffmark" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/W3C-20150513.html", + "reference": "https://spdx.org/licenses/SSH-OpenSSH.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/W3C-20150513.json", + "detailsUrl": "https://spdx.org/licenses/SSH-OpenSSH.json", "referenceNumber": 187, - "name": "W3C Software Notice and Document License (2015-05-13)", - "licenseId": "W3C-20150513", + "name": "SSH OpenSSH license", + "licenseId": "SSH-OpenSSH", "seeAlso": [ - "https://www.w3.org/Consortium/Legal/2015/copyright-software-and-document" + "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Sleepycat.html", + "reference": "https://spdx.org/licenses/DOC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Sleepycat.json", + "detailsUrl": "https://spdx.org/licenses/DOC.json", "referenceNumber": 188, - "name": "Sleepycat License", - "licenseId": "Sleepycat", + "name": "DOC License", + "licenseId": "DOC", "seeAlso": [ - "https://opensource.org/licenses/Sleepycat" + "http://www.cs.wustl.edu/~schmidt/ACE-copying.html", + "https://www.dre.vanderbilt.edu/~schmidt/ACE-copying.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/JasPer-2.0.html", + "reference": "https://spdx.org/licenses/OSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/JasPer-2.0.json", + "detailsUrl": "https://spdx.org/licenses/OSL-1.0.json", "referenceNumber": 189, - "name": "JasPer License", - "licenseId": "JasPer-2.0", + "name": "Open Software License 1.0", + "licenseId": "OSL-1.0", "seeAlso": [ - "http://www.ece.uvic.ca/~mdadams/jasper/LICENSE" + "https://opensource.org/licenses/OSL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Nokia.html", + "reference": "https://spdx.org/licenses/Xnet.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Nokia.json", + "detailsUrl": "https://spdx.org/licenses/Xnet.json", "referenceNumber": 190, - "name": "Nokia Open Source License", - "licenseId": "Nokia", + "name": "X.Net License", + "licenseId": "Xnet", "seeAlso": [ - "https://opensource.org/licenses/nokia" + "https://opensource.org/licenses/Xnet" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/LGPL-2.1+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.1+.json", + "reference": "https://spdx.org/licenses/CDL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CDL-1.0.json", "referenceNumber": 191, - "name": "GNU Library General Public License v2.1 or later", - "licenseId": "LGPL-2.1+", + "name": "Common Documentation License 1.0", + "licenseId": "CDL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", - "https://opensource.org/licenses/LGPL-2.1" + "http://www.opensource.apple.com/cdl/", + "https://fedoraproject.org/wiki/Licensing/Common_Documentation_License", + "https://www.gnu.org/licenses/license-list.html#ACDL" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGL-UK-2.0.html", + "reference": "https://spdx.org/licenses/Latex2e.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-UK-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Latex2e.json", "referenceNumber": 192, - "name": "Open Government Licence v2.0", - "licenseId": "OGL-UK-2.0", + "name": "Latex2e License", + "licenseId": "Latex2e", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" + "https://fedoraproject.org/wiki/Licensing/Latex2e" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/RPL-1.1.html", + "reference": "https://spdx.org/licenses/GPL-1.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/GPL-1.0-or-later.json", "referenceNumber": 193, - "name": "Reciprocal Public License 1.1", - "licenseId": "RPL-1.1", + "name": "GNU General Public License v1.0 or later", + "licenseId": "GPL-1.0-or-later", "seeAlso": [ - "https://opensource.org/licenses/RPL-1.1" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SHL-0.51.html", + "reference": "https://spdx.org/licenses/ISC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SHL-0.51.json", + "detailsUrl": "https://spdx.org/licenses/ISC.json", "referenceNumber": 194, - "name": "Solderpad Hardware License, Version 0.51", - "licenseId": "SHL-0.51", + "name": "ISC License", + "licenseId": "ISC", "seeAlso": [ - "https://solderpad.org/licenses/SHL-0.51/" + "https://www.isc.org/licenses/", + "https://www.isc.org/downloads/software-support-policy/isc-license/", + "https://opensource.org/licenses/ISC" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LiLiQ-P-1.1.html", + "reference": "https://spdx.org/licenses/Xerox.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LiLiQ-P-1.1.json", + "detailsUrl": "https://spdx.org/licenses/Xerox.json", "referenceNumber": 195, - "name": "Licence Libre du Québec – Permissive version 1.1", - "licenseId": "LiLiQ-P-1.1", + "name": "Xerox License", + "licenseId": "Xerox", "seeAlso": [ - "https://forge.gouv.qc.ca/licence/fr/liliq-v1-1/", - "http://opensource.org/licenses/LiLiQ-P-1.1" + "https://fedoraproject.org/wiki/Licensing/Xerox" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OpenSSL.html", + "reference": "https://spdx.org/licenses/Artistic-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OpenSSL.json", + "detailsUrl": "https://spdx.org/licenses/Artistic-1.0.json", "referenceNumber": 196, - "name": "OpenSSL License", - "licenseId": "OpenSSL", + "name": "Artistic License 1.0", + "licenseId": "Artistic-1.0", "seeAlso": [ - "http://www.openssl.org/source/license.html" + "https://opensource.org/licenses/Artistic-1.0" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/C-UDA-1.0.html", + "reference": "https://spdx.org/licenses/CUA-OPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/C-UDA-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CUA-OPL-1.0.json", "referenceNumber": 197, - "name": "Computational Use of Data Agreement v1.0", - "licenseId": "C-UDA-1.0", + "name": "CUA Office Public License v1.0", + "licenseId": "CUA-OPL-1.0", "seeAlso": [ - "https://github.com/microsoft/Computational-Use-of-Data-Agreement/blob/master/C-UDA-1.0.md", - "https://cdla.dev/computational-use-of-data-agreement-v1-0/" + "https://opensource.org/licenses/CUA-OPL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Xnet.html", + "reference": "https://spdx.org/licenses/NPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Xnet.json", + "detailsUrl": "https://spdx.org/licenses/NPL-1.1.json", "referenceNumber": 198, - "name": "X.Net License", - "licenseId": "Xnet", + "name": "Netscape Public License v1.1", + "licenseId": "NPL-1.1", "seeAlso": [ - "https://opensource.org/licenses/Xnet" + "http://www.mozilla.org/MPL/NPL/1.1/" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/APSL-2.0.html", + "reference": "https://spdx.org/licenses/MITNFA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/MITNFA.json", "referenceNumber": 199, - "name": "Apple Public Source License 2.0", - "licenseId": "APSL-2.0", + "name": "MIT +no-false-attribs license", + "licenseId": "MITNFA", "seeAlso": [ - "http://www.opensource.apple.com/license/apsl/" + "https://fedoraproject.org/wiki/Licensing/MITNFA" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FreeBSD-DOC.html", + "reference": "https://spdx.org/licenses/OLDAP-1.4.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FreeBSD-DOC.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.4.json", "referenceNumber": 200, - "name": "FreeBSD Documentation License", - "licenseId": "FreeBSD-DOC", + "name": "Open LDAP Public License v1.4", + "licenseId": "OLDAP-1.4", "seeAlso": [ - "https://www.freebsd.org/copyright/freebsd-doc-license/" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dc9f95c2f3f2ffb5e0ae55fe7388af75547660941" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NPL-1.0.html", + "reference": "https://spdx.org/licenses/Leptonica.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Leptonica.json", "referenceNumber": 201, - "name": "Netscape Public License v1.0", - "licenseId": "NPL-1.0", + "name": "Leptonica License", + "licenseId": "Leptonica", "seeAlso": [ - "http://www.mozilla.org/MPL/NPL/1.0/" + "https://fedoraproject.org/wiki/Licensing/Leptonica" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-or-later.html", + "reference": "https://spdx.org/licenses/OCCT-PL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-or-later.json", + "detailsUrl": "https://spdx.org/licenses/OCCT-PL.json", "referenceNumber": 202, - "name": "GNU General Public License v2.0 or later", - "licenseId": "GPL-2.0-or-later", + "name": "Open CASCADE Technology Public License", + "licenseId": "OCCT-PL", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "http://www.opencascade.com/content/occt-public-license" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CECILL-2.0.html", + "reference": "https://spdx.org/licenses/Qhull.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Qhull.json", "referenceNumber": 203, - "name": "CeCILL Free Software License Agreement v2.0", - "licenseId": "CECILL-2.0", + "name": "Qhull License", + "licenseId": "Qhull", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V2-en.html" + "https://fedoraproject.org/wiki/Licensing/Qhull" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OFL-1.0.html", + "reference": "https://spdx.org/licenses/CPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CPL-1.0.json", "referenceNumber": 204, - "name": "SIL Open Font License 1.0", - "licenseId": "OFL-1.0", + "name": "Common Public License 1.0", + "licenseId": "CPL-1.0", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" + "https://opensource.org/licenses/CPL-1.0" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/EPL-1.0.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.json", "referenceNumber": 205, - "name": "Eclipse Public License 1.0", - "licenseId": "EPL-1.0", + "name": "BSD 3-Clause No Nuclear License", + "licenseId": "BSD-3-Clause-No-Nuclear-License", "seeAlso": [ - "http://www.eclipse.org/legal/epl-v10.html", - "https://opensource.org/licenses/EPL-1.0" + "http://download.oracle.com/otn-pub/java/licenses/bsd.txt?AuthParam\u003d1467140197_43d516ce1776bd08a58235a7785be1cc" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NOSL.html", + "reference": "https://spdx.org/licenses/CC-BY-2.5-AU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NOSL.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5-AU.json", "referenceNumber": 206, - "name": "Netizen Open Source License", - "licenseId": "NOSL", + "name": "Creative Commons Attribution 2.5 Australia", + "licenseId": "CC-BY-2.5-AU", "seeAlso": [ - "http://bits.netizen.com.au/licenses/NOSL/nosl.txt" + "https://creativecommons.org/licenses/by/2.5/au/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MIT-CMU.html", + "reference": "https://spdx.org/licenses/OLDAP-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-CMU.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.1.json", "referenceNumber": 207, - "name": "CMU License", - "licenseId": "MIT-CMU", + "name": "Open LDAP Public License v2.1", + "licenseId": "OLDAP-2.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:MIT?rd\u003dLicensing/MIT#CMU_Style", - "https://github.com/python-pillow/Pillow/blob/fffb426092c8db24a5f4b6df243a8a3c01fb63cd/LICENSE" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db0d176738e96a0d3b9f85cb51e140a86f21be715" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CERN-OHL-S-2.0.html", + "reference": "https://spdx.org/licenses/GPL-2.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-S-2.0.json", + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-only.json", "referenceNumber": 208, - "name": "CERN Open Hardware Licence Version 2 - Strongly Reciprocal", - "licenseId": "CERN-OHL-S-2.0", + "name": "GNU General Public License v2.0 only", + "licenseId": "GPL-2.0-only", "seeAlso": [ - "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SWL.html", + "reference": "https://spdx.org/licenses/ImageMagick.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SWL.json", + "detailsUrl": "https://spdx.org/licenses/ImageMagick.json", "referenceNumber": 209, - "name": "Scheme Widget Library (SWL) Software License Agreement", - "licenseId": "SWL", + "name": "ImageMagick License", + "licenseId": "ImageMagick", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/SWL" + "http://www.imagemagick.org/script/license.php" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-1.0.json", "referenceNumber": 210, - "name": "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", - "licenseId": "CC-BY-NC-SA-2.5", + "name": "Creative Commons Attribution Share Alike 1.0 Generic", + "licenseId": "CC-BY-SA-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.5/legalcode" + "https://creativecommons.org/licenses/by-sa/1.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Zlib.html", + "reference": "https://spdx.org/licenses/LPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zlib.json", + "detailsUrl": "https://spdx.org/licenses/LPL-1.0.json", "referenceNumber": 211, - "name": "zlib License", - "licenseId": "Zlib", + "name": "Lucent Public License Version 1.0", + "licenseId": "LPL-1.0", "seeAlso": [ - "http://www.zlib.net/zlib_license.html", - "https://opensource.org/licenses/Zlib" + "https://opensource.org/licenses/LPL-1.0" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.html", + "reference": "https://spdx.org/licenses/NCSA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.json", + "detailsUrl": "https://spdx.org/licenses/NCSA.json", "referenceNumber": 212, - "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", - "licenseId": "CC-BY-NC-SA-3.0", + "name": "University of Illinois/NCSA Open Source License", + "licenseId": "NCSA", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/3.0/legalcode" + "http://otm.illinois.edu/uiuc_openSource", + "https://opensource.org/licenses/NCSA" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/iMatix.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/iMatix.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause.json", "referenceNumber": 213, - "name": "iMatix Standard Function Library Agreement", - "licenseId": "iMatix", + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "licenseId": "BSD-3-Clause", "seeAlso": [ - "http://legacy.imatix.com/html/sfl/sfl4.htm#license" + "https://opensource.org/licenses/BSD-3-Clause" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-2.5.html", + "reference": "https://spdx.org/licenses/APSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.5.json", + "detailsUrl": "https://spdx.org/licenses/APSL-1.1.json", "referenceNumber": 214, - "name": "Creative Commons Attribution No Derivatives 2.5 Generic", - "licenseId": "CC-BY-ND-2.5", + "name": "Apple Public Source License 1.1", + "licenseId": "APSL-1.1", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/2.5/legalcode" + "http://www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/PHP-3.0.html", + "reference": "https://spdx.org/licenses/Python-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PHP-3.0.json", + "detailsUrl": "https://spdx.org/licenses/Python-2.0.json", "referenceNumber": 215, - "name": "PHP License v3.0", - "licenseId": "PHP-3.0", + "name": "Python License 2.0", + "licenseId": "Python-2.0", "seeAlso": [ - "http://www.php.net/license/3_0.txt", - "https://opensource.org/licenses/PHP-3.0" + "https://opensource.org/licenses/Python-2.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Barr.html", + "reference": "https://spdx.org/licenses/RPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Barr.json", + "detailsUrl": "https://spdx.org/licenses/RPL-1.1.json", "referenceNumber": 216, - "name": "Barr License", - "licenseId": "Barr", + "name": "Reciprocal Public License 1.1", + "licenseId": "RPL-1.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Barr" + "https://opensource.org/licenses/RPL-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/curl.html", + "reference": "https://spdx.org/licenses/CPOL-1.02.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/curl.json", + "detailsUrl": "https://spdx.org/licenses/CPOL-1.02.json", "referenceNumber": 217, - "name": "curl License", - "licenseId": "curl", + "name": "Code Project Open License 1.02", + "licenseId": "CPOL-1.02", "seeAlso": [ - "https://github.com/bagder/curl/blob/master/COPYING" + "http://www.codeproject.com/info/cpol10.aspx" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/Zimbra-1.4.html", + "reference": "https://spdx.org/licenses/YPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zimbra-1.4.json", + "detailsUrl": "https://spdx.org/licenses/YPL-1.0.json", "referenceNumber": 218, - "name": "Zimbra Public License v1.4", - "licenseId": "Zimbra-1.4", + "name": "Yahoo! Public License v1.0", + "licenseId": "YPL-1.0", "seeAlso": [ - "http://www.zimbra.com/legal/zimbra-public-license-1-4" + "http://www.zimbra.com/license/yahoo_public_license_1.0.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TOSL.html", + "reference": "https://spdx.org/licenses/AMDPLPA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TOSL.json", + "detailsUrl": "https://spdx.org/licenses/AMDPLPA.json", "referenceNumber": 219, - "name": "Trusster Open Source License", - "licenseId": "TOSL", + "name": "AMD\u0027s plpa_map.c License", + "licenseId": "AMDPLPA", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/TOSL" + "https://fedoraproject.org/wiki/Licensing/AMD_plpa_map_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.json", + "reference": "https://spdx.org/licenses/GPL-3.0-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-only.json", "referenceNumber": 220, - "name": "GNU General Public License v3.0 w/GCC Runtime Library exception", - "licenseId": "GPL-3.0-with-GCC-exception", + "name": "GNU General Public License v3.0 only", + "licenseId": "GPL-3.0-only", "seeAlso": [ - "https://www.gnu.org/licenses/gcc-exception-3.1.html" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/AFL-1.2.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-1.2.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-or-later.json", "referenceNumber": 221, - "name": "Academic Free License v1.2", - "licenseId": "AFL-1.2", + "name": "GNU Free Documentation License v1.1 or later", + "licenseId": "GFDL-1.1-or-later", "seeAlso": [ - "http://opensource.linux-mirror.org/licenses/afl-1.2.txt", - "http://wayback.archive.org/web/20021204204652/http://www.opensource.org/licenses/academic.php" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.2.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2.json", + "reference": "https://spdx.org/licenses/Info-ZIP.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Info-ZIP.json", "referenceNumber": 222, - "name": "GNU Free Documentation License v1.2", - "licenseId": "GFDL-1.2", + "name": "Info-ZIP License", + "licenseId": "Info-ZIP", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "http://www.info-zip.org/license.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.html", + "reference": "https://spdx.org/licenses/OGDL-Taiwan-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.json", + "detailsUrl": "https://spdx.org/licenses/OGDL-Taiwan-1.0.json", "referenceNumber": 223, - "name": "Creative Commons Attribution No Derivatives 3.0 Germany", - "licenseId": "CC-BY-ND-3.0-DE", + "name": "Taiwan Open Government Data License, version 1.0", + "licenseId": "OGDL-Taiwan-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/3.0/de/legalcode" + "https://data.gov.tw/license" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.html", + "reference": "https://spdx.org/licenses/Unicode-DFS-2015.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2015.json", "referenceNumber": 224, - "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", - "licenseId": "CC-BY-NC-SA-2.0", + "name": "Unicode License Agreement - Data Files and Software (2015)", + "licenseId": "Unicode-DFS-2015", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.0/legalcode" + "https://web.archive.org/web/20151224134844/http://unicode.org/copyright.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-only.html", + "reference": "https://spdx.org/licenses/Python-2.0.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-only.json", + "detailsUrl": "https://spdx.org/licenses/Python-2.0.1.json", "referenceNumber": 225, - "name": "GNU Free Documentation License v1.3 only - invariants", - "licenseId": "GFDL-1.3-invariants-only", + "name": "Python License 2.0.1", + "licenseId": "Python-2.0.1", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://www.python.org/download/releases/2.0.1/license/", + "https://docs.python.org/3/license.html", + "https://github.com/python/cpython/blob/main/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GD.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GD.json", + "reference": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.json", "referenceNumber": 226, - "name": "GD License", - "licenseId": "GD", + "name": "BSD 2-Clause NetBSD License", + "licenseId": "BSD-2-Clause-NetBSD", "seeAlso": [ - "https://libgd.github.io/manuals/2.3.0/files/license-txt.html" + "http://www.netbsd.org/about/redistribution.html#default" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.3.html", + "reference": "https://spdx.org/licenses/LGPL-2.1-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.3.json", + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-only.json", "referenceNumber": 227, - "name": "Open LDAP Public License v2.3", - "licenseId": "OLDAP-2.3", + "name": "GNU Lesser General Public License v2.1 only", + "licenseId": "LGPL-2.1-only", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dd32cf54a32d581ab475d23c810b0a7fbaf8d63c3" + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Unicode-DFS-2016.html", + "reference": "https://spdx.org/licenses/GL2PS.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2016.json", + "detailsUrl": "https://spdx.org/licenses/GL2PS.json", "referenceNumber": 228, - "name": "Unicode License Agreement - Data Files and Software (2016)", - "licenseId": "Unicode-DFS-2016", + "name": "GL2PS License", + "licenseId": "GL2PS", "seeAlso": [ - "http://www.unicode.org/copyright.html" + "http://www.geuz.org/gl2ps/COPYING.GL2PS" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/IBM-pibs.html", + "reference": "https://spdx.org/licenses/TU-Berlin-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IBM-pibs.json", + "detailsUrl": "https://spdx.org/licenses/TU-Berlin-1.0.json", "referenceNumber": 229, - "name": "IBM PowerPC Initialization and Boot Software", - "licenseId": "IBM-pibs", + "name": "Technische Universitaet Berlin License 1.0", + "licenseId": "TU-Berlin-1.0", "seeAlso": [ - "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003darch/powerpc/cpu/ppc4xx/miiphy.c;h\u003d297155fdafa064b955e53e9832de93bfb0cfb85b;hb\u003d9fab4bf4cc077c21e43941866f3f2c196f28670d" + "https://github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FDK-AAC.html", + "reference": "https://spdx.org/licenses/DSDP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FDK-AAC.json", + "detailsUrl": "https://spdx.org/licenses/DSDP.json", "referenceNumber": 230, - "name": "Fraunhofer FDK AAC Codec Library", - "licenseId": "FDK-AAC", + "name": "DSDP License", + "licenseId": "DSDP", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/FDK-AAC", - "https://directory.fsf.org/wiki/License:Fdk" + "https://fedoraproject.org/wiki/Licensing/DSDP" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CATOSL-1.1.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CATOSL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.json", "referenceNumber": 231, - "name": "Computer Associates Trusted Open Source License 1.1", - "licenseId": "CATOSL-1.1", + "name": "GNU Free Documentation License v1.3 or later - invariants", + "licenseId": "GFDL-1.3-invariants-or-later", "seeAlso": [ - "https://opensource.org/licenses/CATOSL-1.1" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.html", + "reference": "https://spdx.org/licenses/Unlicense.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Unlicense.json", "referenceNumber": 232, - "name": "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", - "licenseId": "CC-BY-NC-ND-1.0", + "name": "The Unlicense", + "licenseId": "Unlicense", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd-nc/1.0/legalcode" + "https://unlicense.org/" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/DSDP.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/DSDP.json", + "reference": "https://spdx.org/licenses/GFDL-1.2.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2.json", "referenceNumber": 233, - "name": "DSDP License", - "licenseId": "DSDP", + "name": "GNU Free Documentation License v1.2", + "licenseId": "GFDL-1.2", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/DSDP" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-Patent.html", + "reference": "https://spdx.org/licenses/BitTorrent-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Patent.json", + "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.1.json", "referenceNumber": 234, - "name": "BSD-2-Clause Plus Patent License", - "licenseId": "BSD-2-Clause-Patent", + "name": "BitTorrent Open Source License v1.1", + "licenseId": "BitTorrent-1.1", "seeAlso": [ - "https://opensource.org/licenses/BSDplusPatent" + "http://directory.fsf.org/wiki/License:BitTorrentOSL1.1" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SugarCRM-1.1.3.html", + "reference": "https://spdx.org/licenses/TCP-wrappers.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SugarCRM-1.1.3.json", + "detailsUrl": "https://spdx.org/licenses/TCP-wrappers.json", "referenceNumber": 235, - "name": "SugarCRM Public License v1.1.3", - "licenseId": "SugarCRM-1.1.3", + "name": "TCP Wrappers License", + "licenseId": "TCP-wrappers", "seeAlso": [ - "http://www.sugarcrm.com/crm/SPL" + "http://rc.quest.com/topics/openssh/license.php#tcpwrappers" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.html", + "reference": "https://spdx.org/licenses/psutils.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json", + "detailsUrl": "https://spdx.org/licenses/psutils.json", "referenceNumber": 236, - "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", - "licenseId": "CC-BY-NC-SA-2.0-UK", + "name": "psutils License", + "licenseId": "psutils", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.0/uk/legalcode" + "https://fedoraproject.org/wiki/Licensing/psutils" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-2.5.html", + "reference": "https://spdx.org/licenses/Abstyles.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.5.json", + "detailsUrl": "https://spdx.org/licenses/Abstyles.json", "referenceNumber": 237, - "name": "Creative Commons Attribution Non Commercial 2.5 Generic", - "licenseId": "CC-BY-NC-2.5", + "name": "Abstyles License", + "licenseId": "Abstyles", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/2.5/legalcode" + "https://fedoraproject.org/wiki/Licensing/Abstyles" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Unlicense.html", + "reference": "https://spdx.org/licenses/Plexus.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unlicense.json", + "detailsUrl": "https://spdx.org/licenses/Plexus.json", "referenceNumber": 238, - "name": "The Unlicense", - "licenseId": "Unlicense", + "name": "Plexus Classworlds License", + "licenseId": "Plexus", "seeAlso": [ - "https://unlicense.org/" + "https://fedoraproject.org/wiki/Licensing/Plexus_Classworlds_License" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AGPL-1.0-or-later.html", + "reference": "https://spdx.org/licenses/MIT-0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-or-later.json", + "detailsUrl": "https://spdx.org/licenses/MIT-0.json", "referenceNumber": 239, - "name": "Affero General Public License v1.0 or later", - "licenseId": "AGPL-1.0-or-later", + "name": "MIT No Attribution", + "licenseId": "MIT-0", "seeAlso": [ - "http://www.affero.org/oagpl.html" + "https://github.com/aws/mit-0", + "https://romanrm.net/mit-zero", + "https://github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GL2PS.html", + "reference": "https://spdx.org/licenses/Zend-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GL2PS.json", + "detailsUrl": "https://spdx.org/licenses/Zend-2.0.json", "referenceNumber": 240, - "name": "GL2PS License", - "licenseId": "GL2PS", + "name": "Zend License v2.0", + "licenseId": "Zend-2.0", "seeAlso": [ - "http://www.geuz.org/gl2ps/COPYING.GL2PS" + "https://web.archive.org/web/20130517195954/http://www.zend.com/license/2_00.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/zlib-acknowledgement.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/zlib-acknowledgement.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-or-later.json", "referenceNumber": 241, - "name": "zlib/libpng License with Acknowledgement", - "licenseId": "zlib-acknowledgement", + "name": "GNU Free Documentation License v1.3 or later", + "licenseId": "GFDL-1.3-or-later", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/ZlibWithAcknowledgement" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/TU-Berlin-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TU-Berlin-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.json", "referenceNumber": 242, - "name": "Technische Universitaet Berlin License 1.0", - "licenseId": "TU-Berlin-1.0", + "name": "Creative Commons Attribution Share Alike 2.1 Japan", + "licenseId": "CC-BY-SA-2.1-JP", "seeAlso": [ - "https://github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT" + "https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Apache-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-NL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Apache-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-NL.json", "referenceNumber": 243, - "name": "Apache License 1.0", - "licenseId": "Apache-1.0", + "name": "Creative Commons Attribution 3.0 Netherlands", + "licenseId": "CC-BY-3.0-NL", "seeAlso": [ - "http://www.apache.org/licenses/LICENSE-1.0" + "https://creativecommons.org/licenses/by/3.0/nl/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Crossword.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Crossword.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.json", "referenceNumber": 244, - "name": "Crossword License", - "licenseId": "Crossword", + "name": "Creative Commons Attribution Share Alike 2.0 England and Wales", + "licenseId": "CC-BY-SA-2.0-UK", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Crossword" + "https://creativecommons.org/licenses/by-sa/2.0/uk/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/HPND-sell-variant.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/HPND-sell-variant.json", + "reference": "https://spdx.org/licenses/eCos-2.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/eCos-2.0.json", "referenceNumber": 245, - "name": "Historical Permission Notice and Disclaimer - sell variant", - "licenseId": "HPND-sell-variant", + "name": "eCos license version 2.0", + "licenseId": "eCos-2.0", "seeAlso": [ - "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h\u003dv4.19" + "https://www.gnu.org/licenses/ecos-license.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause.html", + "reference": "https://spdx.org/licenses/Elastic-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause.json", + "detailsUrl": "https://spdx.org/licenses/Elastic-2.0.json", "referenceNumber": 246, - "name": "BSD 2-Clause \"Simplified\" License", - "licenseId": "BSD-2-Clause", + "name": "Elastic License 2.0", + "licenseId": "Elastic-2.0", "seeAlso": [ - "https://opensource.org/licenses/BSD-2-Clause" + "https://www.elastic.co/licensing/elastic-license", + "https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE-2.0.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.json", + "reference": "https://spdx.org/licenses/Nunit.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/Nunit.json", "referenceNumber": 247, - "name": "Creative Commons Attribution Share Alike 3.0 Germany", - "licenseId": "CC-BY-SA-3.0-DE", + "name": "Nunit License", + "licenseId": "Nunit", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/3.0/de/legalcode" + "https://fedoraproject.org/wiki/Licensing/Nunit" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/AMPAS.html", + "reference": "https://spdx.org/licenses/SISSL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AMPAS.json", + "detailsUrl": "https://spdx.org/licenses/SISSL-1.2.json", "referenceNumber": 248, - "name": "Academy of Motion Picture Arts and Sciences BSD", - "licenseId": "AMPAS", + "name": "Sun Industry Standards Source License v1.2", + "licenseId": "SISSL-1.2", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/BSD#AMPASBSD" + "http://gridscheduler.sourceforge.net/Gridengine_SISSL_license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/YPL-1.1.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Attribution.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/YPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Attribution.json", "referenceNumber": 249, - "name": "Yahoo! Public License v1.1", - "licenseId": "YPL-1.1", + "name": "BSD with attribution", + "licenseId": "BSD-3-Clause-Attribution", "seeAlso": [ - "http://www.zimbra.com/license/yahoo_public_license_1.1.html" + "https://fedoraproject.org/wiki/Licensing/BSD_with_Attribution" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.html", + "reference": "https://spdx.org/licenses/EPL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.json", + "detailsUrl": "https://spdx.org/licenses/EPL-2.0.json", "referenceNumber": 250, - "name": "Creative Commons Attribution Non Commercial Share Alike 4.0 International", - "licenseId": "CC-BY-NC-SA-4.0", + "name": "Eclipse Public License 2.0", + "licenseId": "EPL-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode" + "https://www.eclipse.org/legal/epl-2.0", + "https://www.opensource.org/licenses/EPL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Glide.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Glide.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.json", "referenceNumber": 251, - "name": "3dfx Glide License", - "licenseId": "Glide", + "name": "GNU Free Documentation License v1.1 or later - invariants", + "licenseId": "GFDL-1.1-invariants-or-later", "seeAlso": [ - "http://www.users.on.net/~triforce/glidexp/COPYING.txt" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Artistic-1.0.html", + "reference": "https://spdx.org/licenses/GPL-3.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Artistic-1.0.json", + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-or-later.json", "referenceNumber": 252, - "name": "Artistic License 1.0", - "licenseId": "Artistic-1.0", - "seeAlso": [ - "https://opensource.org/licenses/Artistic-1.0" - ], - "isOsiApproved": true, - "isFsfLibre": false - }, - { - "reference": "https://spdx.org/licenses/OSL-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-1.0.json", - "referenceNumber": 253, - "name": "Open Software License 1.0", - "licenseId": "OSL-1.0", + "name": "GNU General Public License v3.0 or later", + "licenseId": "GPL-3.0-or-later", "seeAlso": [ - "https://opensource.org/licenses/OSL-1.0" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/0BSD.html", + "reference": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/0BSD.json", - "referenceNumber": 254, - "name": "BSD Zero Clause License", - "licenseId": "0BSD", + "detailsUrl": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.json", + "referenceNumber": 253, + "name": "Cryptographic Autonomy License 1.0 (Combined Work Exception)", + "licenseId": "CAL-1.0-Combined-Work-Exception", "seeAlso": [ - "http://landley.net/toybox/license.html", - "https://opensource.org/licenses/0BSD" + "http://cryptographicautonomylicense.com/license-text.html", + "https://opensource.org/licenses/CAL-1.0" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.7.html", + "reference": "https://spdx.org/licenses/LPPL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.7.json", - "referenceNumber": 255, - "name": "Open LDAP Public License v2.7", - "licenseId": "OLDAP-2.7", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.2.json", + "referenceNumber": 254, + "name": "LaTeX Project Public License v1.2", + "licenseId": "LPPL-1.2", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d47c2415c1df81556eeb39be6cad458ef87c534a2" + "http://www.latex-project.org/lppl/lppl-1-2.txt" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/TAPR-OHL-1.0.html", + "reference": "https://spdx.org/licenses/CNRI-Jython.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TAPR-OHL-1.0.json", - "referenceNumber": 256, - "name": "TAPR Open Hardware License v1.0", - "licenseId": "TAPR-OHL-1.0", + "detailsUrl": "https://spdx.org/licenses/CNRI-Jython.json", + "referenceNumber": 255, + "name": "CNRI Jython License", + "licenseId": "CNRI-Jython", "seeAlso": [ - "https://www.tapr.org/OHL" + "http://www.jython.org/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Adobe-2006.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Adobe-2006.json", - "referenceNumber": 257, - "name": "Adobe Systems Incorporated Source Code License Agreement", - "licenseId": "Adobe-2006", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-4.0.json", + "referenceNumber": 256, + "name": "Creative Commons Attribution Non Commercial 4.0 International", + "licenseId": "CC-BY-NC-4.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AdobeLicense" + "https://creativecommons.org/licenses/by-nc/4.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { "reference": "https://spdx.org/licenses/AFL-1.1.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/AFL-1.1.json", - "referenceNumber": 258, + "referenceNumber": 257, "name": "Academic Free License v1.1", "licenseId": "AFL-1.1", "seeAlso": [ @@ -3263,2695 +3252,2919 @@ "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Spencer-94.html", + "reference": "https://spdx.org/licenses/ANTLR-PD-fallback.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Spencer-94.json", - "referenceNumber": 259, - "name": "Spencer License 94", - "licenseId": "Spencer-94", + "detailsUrl": "https://spdx.org/licenses/ANTLR-PD-fallback.json", + "referenceNumber": 258, + "name": "ANTLR Software Rights Notice with license fallback", + "licenseId": "ANTLR-PD-fallback", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" + "http://www.antlr2.org/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/XSkat.html", + "reference": "https://spdx.org/licenses/AFL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/XSkat.json", + "detailsUrl": "https://spdx.org/licenses/AFL-1.2.json", + "referenceNumber": 259, + "name": "Academic Free License v1.2", + "licenseId": "AFL-1.2", + "seeAlso": [ + "http://opensource.linux-mirror.org/licenses/afl-1.2.txt", + "http://wayback.archive.org/web/20021204204652/http://www.opensource.org/licenses/academic.php" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/NLOD-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NLOD-1.0.json", "referenceNumber": 260, - "name": "XSkat License", - "licenseId": "XSkat", + "name": "Norwegian Licence for Open Government Data (NLOD) 1.0", + "licenseId": "NLOD-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/XSkat_License" + "http://data.norge.no/nlod/en/1.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Newsletr.html", + "reference": "https://spdx.org/licenses/HTMLTIDY.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Newsletr.json", + "detailsUrl": "https://spdx.org/licenses/HTMLTIDY.json", "referenceNumber": 261, - "name": "Newsletr License", - "licenseId": "Newsletr", + "name": "HTML Tidy License", + "licenseId": "HTMLTIDY", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Newsletr" + "https://github.com/htacg/tidy-html5/blob/next/README/LICENSE.md" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-2.0.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-DE.json", "referenceNumber": 262, - "name": "Creative Commons Attribution Share Alike 2.0 Generic", - "licenseId": "CC-BY-SA-2.0", + "name": "Creative Commons Attribution 3.0 Germany", + "licenseId": "CC-BY-3.0-DE", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/2.0/legalcode" + "https://creativecommons.org/licenses/by/3.0/de/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ZPL-1.1.html", + "reference": "https://spdx.org/licenses/ECL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ZPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/ECL-1.0.json", "referenceNumber": 263, - "name": "Zope Public License 1.1", - "licenseId": "ZPL-1.1", + "name": "Educational Community License v1.0", + "licenseId": "ECL-1.0", "seeAlso": [ - "http://old.zope.org/Resources/License/ZPL-1.1" + "https://opensource.org/licenses/ECL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-only.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-only.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.json", "referenceNumber": 264, - "name": "GNU Free Documentation License v1.3 only", - "licenseId": "GFDL-1.3-only", + "name": "BSD 3-Clause No Nuclear License 2014", + "licenseId": "BSD-3-Clause-No-Nuclear-License-2014", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://java.net/projects/javaeetutorial/pages/BerkeleyLicense" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EUPL-1.0.html", + "reference": "https://spdx.org/licenses/OFL-1.0-no-RFN.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/OFL-1.0-no-RFN.json", "referenceNumber": 265, - "name": "European Union Public License 1.0", - "licenseId": "EUPL-1.0", + "name": "SIL Open Font License 1.0 with no Reserved Font Name", + "licenseId": "OFL-1.0-no-RFN", "seeAlso": [ - "http://ec.europa.eu/idabc/en/document/7330.html", - "http://ec.europa.eu/idabc/servlets/Doc027f.pdf?id\u003d31096" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Clear.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Clear.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.json", "referenceNumber": 266, - "name": "BSD 3-Clause Clear License", - "licenseId": "BSD-3-Clause-Clear", + "name": "GNU Free Documentation License v1.2 or later - invariants", + "licenseId": "GFDL-1.2-invariants-or-later", "seeAlso": [ - "http://labs.metacarta.com/license-explanation.html#license" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-1.0-or-later.html", + "reference": "https://spdx.org/licenses/CECILL-B.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0-or-later.json", + "detailsUrl": "https://spdx.org/licenses/CECILL-B.json", "referenceNumber": 267, - "name": "GNU General Public License v1.0 or later", - "licenseId": "GPL-1.0-or-later", + "name": "CeCILL-B Free Software License Agreement", + "licenseId": "CECILL-B", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + "http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CNRI-Jython.html", + "reference": "https://spdx.org/licenses/CECILL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CNRI-Jython.json", + "detailsUrl": "https://spdx.org/licenses/CECILL-2.1.json", "referenceNumber": 268, - "name": "CNRI Jython License", - "licenseId": "CNRI-Jython", + "name": "CeCILL Free Software License Agreement v2.1", + "licenseId": "CECILL-2.1", "seeAlso": [ - "http://www.jython.org/license.html" + "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Adobe-Glyph.html", + "reference": "https://spdx.org/licenses/SGI-B-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Adobe-Glyph.json", + "detailsUrl": "https://spdx.org/licenses/SGI-B-1.0.json", "referenceNumber": 269, - "name": "Adobe Glyph List License", - "licenseId": "Adobe-Glyph", + "name": "SGI Free Software License B v1.0", + "licenseId": "SGI-B-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT#AdobeGlyph" + "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.json", + "reference": "https://spdx.org/licenses/NBPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NBPL-1.0.json", "referenceNumber": 270, - "name": "GNU General Public License v2.0 w/Bison exception", - "licenseId": "GPL-2.0-with-bison-exception", + "name": "Net Boolean Public License v1", + "licenseId": "NBPL-1.0", "seeAlso": [ - "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d37b4b3f6cc4bf34e1d3dec61e69914b9819d8894" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MITNFA.html", + "reference": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MITNFA.json", + "detailsUrl": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.json", "referenceNumber": 271, - "name": "MIT +no-false-attribs license", - "licenseId": "MITNFA", + "name": "CNRI Python Open Source GPL Compatible License Agreement", + "licenseId": "CNRI-Python-GPL-Compatible", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MITNFA" + "http://www.python.org/download/releases/1.6.1/download_win/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/APSL-1.0.html", + "reference": "https://spdx.org/licenses/SchemeReport.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/SchemeReport.json", "referenceNumber": 272, - "name": "Apple Public Source License 1.0", - "licenseId": "APSL-1.0", - "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Apple_Public_Source_License_1.0" - ], - "isOsiApproved": true, - "isFsfLibre": false + "name": "Scheme Language Report License", + "licenseId": "SchemeReport", + "seeAlso": [], + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/JPNIC.html", + "reference": "https://spdx.org/licenses/Apache-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/JPNIC.json", + "detailsUrl": "https://spdx.org/licenses/Apache-2.0.json", "referenceNumber": 273, - "name": "Japan Network Information Center License", - "licenseId": "JPNIC", + "name": "Apache License 2.0", + "licenseId": "Apache-2.0", "seeAlso": [ - "https://gitlab.isc.org/isc-projects/bind9/blob/master/COPYRIGHT#L366" + "https://www.apache.org/licenses/LICENSE-2.0", + "https://opensource.org/licenses/Apache-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-Protection.html", + "reference": "https://spdx.org/licenses/ODbL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-Protection.json", + "detailsUrl": "https://spdx.org/licenses/ODbL-1.0.json", "referenceNumber": 274, - "name": "BSD Protection License", - "licenseId": "BSD-Protection", + "name": "Open Data Commons Open Database License v1.0", + "licenseId": "ODbL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/BSD_Protection_License" + "http://www.opendatacommons.org/licenses/odbl/1.0/", + "https://opendatacommons.org/licenses/odbl/1-0/" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OGL-UK-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-UK-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0.json", "referenceNumber": 275, - "name": "Open Government Licence v1.0", - "licenseId": "OGL-UK-1.0", + "name": "Creative Commons Attribution No Derivatives 3.0 Unported", + "licenseId": "CC-BY-ND-3.0", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/1/" + "https://creativecommons.org/licenses/by-nd/3.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/MulanPSL-2.0.html", + "reference": "https://spdx.org/licenses/LPPL-1.3c.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MulanPSL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.3c.json", "referenceNumber": 276, - "name": "Mulan Permissive Software License, Version 2", - "licenseId": "MulanPSL-2.0", + "name": "LaTeX Project Public License v1.3c", + "licenseId": "LPPL-1.3c", "seeAlso": [ - "https://license.coscl.org.cn/MulanPSL2/" + "http://www.latex-project.org/lppl/lppl-1-3c.txt", + "https://opensource.org/licenses/LPPL-1.3c" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/ImageMagick.html", + "reference": "https://spdx.org/licenses/APSL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ImageMagick.json", + "detailsUrl": "https://spdx.org/licenses/APSL-1.2.json", "referenceNumber": 277, - "name": "ImageMagick License", - "licenseId": "ImageMagick", + "name": "Apple Public Source License 1.2", + "licenseId": "APSL-1.2", "seeAlso": [ - "http://www.imagemagick.org/script/license.php" + "http://www.samurajdata.se/opensource/mirror/licenses/apsl.php" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OSL-2.0.html", + "reference": "https://spdx.org/licenses/OFL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/OFL-1.1.json", "referenceNumber": 278, - "name": "Open Software License 2.0", - "licenseId": "OSL-2.0", + "name": "SIL Open Font License 1.1", + "licenseId": "OFL-1.1", "seeAlso": [ - "http://web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", + "https://opensource.org/licenses/OFL-1.1" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.0.html", + "reference": "https://spdx.org/licenses/MS-PL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.json", + "detailsUrl": "https://spdx.org/licenses/MS-PL.json", "referenceNumber": 279, - "name": "Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B)", - "licenseId": "OLDAP-2.0", + "name": "Microsoft Public License", + "licenseId": "MS-PL", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcbf50f4e1185a21abd4c0a54d3f4341fe28f36ea" + "http://www.microsoft.com/opensource/licenses.mspx", + "https://opensource.org/licenses/MS-PL" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/bzip2-1.0.6.html", + "reference": "https://spdx.org/licenses/C-UDA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.6.json", + "detailsUrl": "https://spdx.org/licenses/C-UDA-1.0.json", "referenceNumber": 280, - "name": "bzip2 and libbzip2 License v1.0.6", - "licenseId": "bzip2-1.0.6", + "name": "Computational Use of Data Agreement v1.0", + "licenseId": "C-UDA-1.0", "seeAlso": [ - "https://sourceware.org/git/?p\u003dbzip2.git;a\u003dblob;f\u003dLICENSE;hb\u003dbzip2-1.0.6", - "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" + "https://github.com/microsoft/Computational-Use-of-Data-Agreement/blob/master/C-UDA-1.0.md", + "https://cdla.dev/computational-use-of-data-agreement-v1-0/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Eurosym.html", + "reference": "https://spdx.org/licenses/Sendmail.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Eurosym.json", + "detailsUrl": "https://spdx.org/licenses/Sendmail.json", "referenceNumber": 281, - "name": "Eurosym License", - "licenseId": "Eurosym", + "name": "Sendmail License", + "licenseId": "Sendmail", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Eurosym" + "http://www.sendmail.com/pdfs/open_source/sendmail_license.pdf", + "https://web.archive.org/web/20160322142305/https://www.sendmail.com/pdfs/open_source/sendmail_license.pdf" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/JSON.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/JSON.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json", "referenceNumber": 282, - "name": "JSON License", - "licenseId": "JSON", + "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + "licenseId": "CC-BY-NC-SA-3.0-DE", "seeAlso": [ - "http://www.json.org/license.html" + "https://creativecommons.org/licenses/by-nc-sa/3.0/de/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json", + "reference": "https://spdx.org/licenses/GPL-2.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0.json", "referenceNumber": 283, - "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", - "licenseId": "CC-BY-NC-ND-3.0-IGO", + "name": "GNU General Public License v2.0 only", + "licenseId": "GPL-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/3.0/igo/legalcode" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LPL-1.0.html", + "reference": "https://spdx.org/licenses/libselinux-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/libselinux-1.0.json", "referenceNumber": 284, - "name": "Lucent Public License Version 1.0", - "licenseId": "LPL-1.0", + "name": "libselinux public domain notice", + "licenseId": "libselinux-1.0", "seeAlso": [ - "https://opensource.org/licenses/LPL-1.0" + "https://github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AGPL-1.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/AGPL-1.0.json", + "reference": "https://spdx.org/licenses/CC-BY-ND-2.5.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.5.json", "referenceNumber": 285, - "name": "Affero General Public License v1.0", - "licenseId": "AGPL-1.0", + "name": "Creative Commons Attribution No Derivatives 2.5 Generic", + "licenseId": "CC-BY-ND-2.5", "seeAlso": [ - "http://www.affero.org/oagpl.html" + "https://creativecommons.org/licenses/by-nd/2.5/legalcode" ], "isOsiApproved": false, - "isFsfLibre": true + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CECILL-2.1.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-2.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0.json", "referenceNumber": 286, - "name": "CeCILL Free Software License Agreement v2.1", - "licenseId": "CECILL-2.1", + "name": "Creative Commons Attribution Non Commercial 3.0 Unported", + "licenseId": "CC-BY-NC-3.0", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" + "https://creativecommons.org/licenses/by-nc/3.0/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/MIT-0.html", + "reference": "https://spdx.org/licenses/MS-LPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-0.json", + "detailsUrl": "https://spdx.org/licenses/MS-LPL.json", "referenceNumber": 287, - "name": "MIT No Attribution", - "licenseId": "MIT-0", + "name": "Microsoft Limited Public License", + "licenseId": "MS-LPL", "seeAlso": [ - "https://github.com/aws/mit-0", - "https://romanrm.net/mit-zero", - "https://github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE" + "https://www.openhub.net/licenses/mslpl", + "https://github.com/gabegundy/atlserver/blob/master/License.txt", + "https://en.wikipedia.org/wiki/Shared_Source_Initiative#Microsoft_Limited_Public_License_(Ms-LPL)" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Spencer-99.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Spencer-99.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.json", "referenceNumber": 288, - "name": "Spencer License 99", - "licenseId": "Spencer-99", + "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + "licenseId": "CC-BY-NC-SA-2.0", "seeAlso": [ - "http://www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c" + "https://creativecommons.org/licenses/by-nc-sa/2.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.html", + "reference": "https://spdx.org/licenses/AMPAS.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.json", + "detailsUrl": "https://spdx.org/licenses/AMPAS.json", "referenceNumber": 289, - "name": "PolyForm Noncommercial License 1.0.0", - "licenseId": "PolyForm-Noncommercial-1.0.0", + "name": "Academy of Motion Picture Arts and Sciences BSD", + "licenseId": "AMPAS", "seeAlso": [ - "https://polyformproject.org/licenses/noncommercial/1.0.0" + "https://fedoraproject.org/wiki/Licensing/BSD#AMPASBSD" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Abstyles.html", + "reference": "https://spdx.org/licenses/AAL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Abstyles.json", + "detailsUrl": "https://spdx.org/licenses/AAL.json", "referenceNumber": 290, - "name": "Abstyles License", - "licenseId": "Abstyles", + "name": "Attribution Assurance License", + "licenseId": "AAL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Abstyles" + "https://opensource.org/licenses/attribution" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-or-later.html", + "reference": "https://spdx.org/licenses/Bahyph.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-or-later.json", + "detailsUrl": "https://spdx.org/licenses/Bahyph.json", "referenceNumber": 291, - "name": "GNU Free Documentation License v1.3 or later", - "licenseId": "GFDL-1.3-or-later", + "name": "Bahyph License", + "licenseId": "Bahyph", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://fedoraproject.org/wiki/Licensing/Bahyph" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NRL.html", + "reference": "https://spdx.org/licenses/Entessa.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NRL.json", + "detailsUrl": "https://spdx.org/licenses/Entessa.json", "referenceNumber": 292, - "name": "NRL License", - "licenseId": "NRL", + "name": "Entessa Public License v1.0", + "licenseId": "Entessa", "seeAlso": [ - "http://web.mit.edu/network/isakmp/nrllicense.html" + "https://opensource.org/licenses/Entessa" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Libpng.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-IGO.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Libpng.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-IGO.json", "referenceNumber": 293, - "name": "libpng License", - "licenseId": "Libpng", + "name": "Creative Commons Attribution 3.0 IGO", + "licenseId": "CC-BY-3.0-IGO", "seeAlso": [ - "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" + "https://creativecommons.org/licenses/by/3.0/igo/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NAIST-2003.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NAIST-2003.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.json", "referenceNumber": 294, - "name": "Nara Institute of Science and Technology License (2003)", - "licenseId": "NAIST-2003", + "name": "GNU Free Documentation License v1.3 or later - no invariants", + "licenseId": "GFDL-1.3-no-invariants-or-later", "seeAlso": [ - "https://enterprise.dejacode.com/licenses/public/naist-2003/#license-text", - "https://github.com/nodejs/node/blob/4a19cc8947b1bba2b2d27816ec3d0edf9b28e503/LICENSE#L343" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/XFree86-1.1.html", + "reference": "https://spdx.org/licenses/IJG.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/XFree86-1.1.json", + "detailsUrl": "https://spdx.org/licenses/IJG.json", "referenceNumber": 295, - "name": "XFree86 License 1.1", - "licenseId": "XFree86-1.1", + "name": "Independent JPEG Group License", + "licenseId": "IJG", "seeAlso": [ - "http://www.xfree86.org/current/LICENSE4.html" + "http://dev.w3.org/cvsweb/Amaya/libjpeg/Attic/README?rev\u003d1.2" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OFL-1.1-RFN.html", + "reference": "https://spdx.org/licenses/CERN-OHL-W-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.1-RFN.json", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-W-2.0.json", "referenceNumber": 296, - "name": "SIL Open Font License 1.1 with Reserved Font Name", - "licenseId": "OFL-1.1-RFN", + "name": "CERN Open Hardware Licence Version 2 - Weakly Reciprocal", + "licenseId": "CERN-OHL-W-2.0", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", - "https://opensource.org/licenses/OFL-1.1" + "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-US.html", + "reference": "https://spdx.org/licenses/SWL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-US.json", + "detailsUrl": "https://spdx.org/licenses/SWL.json", "referenceNumber": 297, - "name": "Creative Commons Attribution 3.0 United States", - "licenseId": "CC-BY-3.0-US", + "name": "Scheme Widget Library (SWL) Software License Agreement", + "licenseId": "SWL", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/us/legalcode" + "https://fedoraproject.org/wiki/Licensing/SWL" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EFL-2.0.html", + "reference": "https://spdx.org/licenses/FSFULLR.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EFL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/FSFULLR.json", "referenceNumber": 298, - "name": "Eiffel Forum License v2.0", - "licenseId": "EFL-2.0", + "name": "FSF Unlimited License (with License Retention)", + "licenseId": "FSFULLR", "seeAlso": [ - "http://www.eiffel-nice.org/license/eiffel-forum-license-2.html", - "https://opensource.org/licenses/EFL-2.0" + "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPPL-1.0.html", + "reference": "https://spdx.org/licenses/TMate.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/TMate.json", "referenceNumber": 299, - "name": "LaTeX Project Public License v1.0", - "licenseId": "LPPL-1.0", + "name": "TMate Open Source License", + "licenseId": "TMate", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-0.txt" + "http://svnkit.com/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/IPL-1.0.html", + "reference": "https://spdx.org/licenses/Artistic-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Artistic-2.0.json", "referenceNumber": 300, - "name": "IBM Public License v1.0", - "licenseId": "IPL-1.0", + "name": "Artistic License 2.0", + "licenseId": "Artistic-2.0", "seeAlso": [ - "https://opensource.org/licenses/IPL-1.0" + "http://www.perlfoundation.org/artistic_license_2_0", + "https://www.perlfoundation.org/artistic-license-20.html", + "https://opensource.org/licenses/artistic-license-2.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/ANTLR-PD.html", + "reference": "https://spdx.org/licenses/APSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ANTLR-PD.json", + "detailsUrl": "https://spdx.org/licenses/APSL-1.0.json", "referenceNumber": 301, - "name": "ANTLR Software Rights Notice", - "licenseId": "ANTLR-PD", + "name": "Apple Public Source License 1.0", + "licenseId": "APSL-1.0", "seeAlso": [ - "http://www.antlr2.org/license.html" + "https://fedoraproject.org/wiki/Licensing/Apple_Public_Source_License_1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.html", + "reference": "https://spdx.org/licenses/ClArtistic.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.json", + "detailsUrl": "https://spdx.org/licenses/ClArtistic.json", "referenceNumber": 302, - "name": "PolyForm Small Business License 1.0.0", - "licenseId": "PolyForm-Small-Business-1.0.0", + "name": "Clarified Artistic License", + "licenseId": "ClArtistic", "seeAlso": [ - "https://polyformproject.org/licenses/small-business/1.0.0" + "http://gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/", + "http://www.ncftp.com/ncftp/doc/LICENSE.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-1.0-only.html", + "reference": "https://spdx.org/licenses/ZPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0-only.json", + "detailsUrl": "https://spdx.org/licenses/ZPL-1.1.json", "referenceNumber": 303, - "name": "GNU General Public License v1.0 only", - "licenseId": "GPL-1.0-only", + "name": "Zope Public License 1.1", + "licenseId": "ZPL-1.1", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + "http://old.zope.org/Resources/License/ZPL-1.1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.html", + "reference": "https://spdx.org/licenses/NLPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.json", + "detailsUrl": "https://spdx.org/licenses/NLPL.json", "referenceNumber": 304, - "name": "Cryptographic Autonomy License 1.0 (Combined Work Exception)", - "licenseId": "CAL-1.0-Combined-Work-Exception", + "name": "No Limit Public License", + "licenseId": "NLPL", "seeAlso": [ - "http://cryptographicautonomylicense.com/license-text.html", - "https://opensource.org/licenses/CAL-1.0" + "https://fedoraproject.org/wiki/Licensing/NLPL" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FreeImage.html", + "reference": "https://spdx.org/licenses/OSL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FreeImage.json", + "detailsUrl": "https://spdx.org/licenses/OSL-2.0.json", "referenceNumber": 305, - "name": "FreeImage Public License v1.0", - "licenseId": "FreeImage", + "name": "Open Software License 2.0", + "licenseId": "OSL-2.0", "seeAlso": [ - "http://freeimage.sourceforge.net/freeimage-license.txt" + "http://web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.json", + "reference": "https://spdx.org/licenses/psfrag.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/psfrag.json", "referenceNumber": 306, - "name": "GNU General Public License v2.0 w/GCC Runtime Library exception", - "licenseId": "GPL-2.0-with-GCC-exception", + "name": "psfrag License", + "licenseId": "psfrag", "seeAlso": [ - "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" + "https://fedoraproject.org/wiki/Licensing/psfrag" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.json", + "reference": "https://spdx.org/licenses/mpi-permissive.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/mpi-permissive.json", "referenceNumber": 307, - "name": "BSD 2-Clause FreeBSD License", - "licenseId": "BSD-2-Clause-FreeBSD", + "name": "mpi Permissive License", + "licenseId": "mpi-permissive", "seeAlso": [ - "http://www.freebsd.org/copyright/freebsd-license.html" + "https://sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl\u003d19#L19" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.2.1.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.json", "referenceNumber": 308, - "name": "Open LDAP Public License v2.2.1", - "licenseId": "OLDAP-2.2.1", + "name": "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + "licenseId": "CC-BY-NC-ND-1.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d4bc786f34b50aa301be6f5600f58a980070f481e" + "https://creativecommons.org/licenses/by-nd-nc/1.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.html", + "reference": "https://spdx.org/licenses/OLDAP-2.2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.1.json", "referenceNumber": 309, - "name": "GNU Free Documentation License v1.2 or later - no invariants", - "licenseId": "GFDL-1.2-no-invariants-or-later", + "name": "Open LDAP Public License v2.2.1", + "licenseId": "OLDAP-2.2.1", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d4bc786f34b50aa301be6f5600f58a980070f481e" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.html", + "reference": "https://spdx.org/licenses/SSPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.json", + "detailsUrl": "https://spdx.org/licenses/SSPL-1.0.json", "referenceNumber": 310, - "name": "GNU Free Documentation License v1.2 only - no invariants", - "licenseId": "GFDL-1.2-no-invariants-only", + "name": "Server Side Public License, v 1", + "licenseId": "SSPL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://www.mongodb.com/licensing/server-side-public-license" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Afmparse.html", + "reference": "https://spdx.org/licenses/Dotseqn.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Afmparse.json", + "detailsUrl": "https://spdx.org/licenses/Dotseqn.json", "referenceNumber": 311, - "name": "Afmparse License", - "licenseId": "Afmparse", + "name": "Dotseqn License", + "licenseId": "Dotseqn", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Afmparse" + "https://fedoraproject.org/wiki/Licensing/Dotseqn" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.2.2.html", + "reference": "https://spdx.org/licenses/SMLNJ.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.2.json", + "detailsUrl": "https://spdx.org/licenses/SMLNJ.json", "referenceNumber": 312, - "name": "Open LDAP Public License 2.2.2", - "licenseId": "OLDAP-2.2.2", - "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003ddf2cc1e21eb7c160695f5b7cffd6296c151ba188" + "name": "Standard ML of New Jersey License", + "licenseId": "SMLNJ", + "seeAlso": [ + "https://www.smlnj.org/license.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Parity-6.0.0.html", + "reference": "https://spdx.org/licenses/OLDAP-2.2.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Parity-6.0.0.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.2.json", "referenceNumber": 313, - "name": "The Parity Public License 6.0.0", - "licenseId": "Parity-6.0.0", + "name": "Open LDAP Public License 2.2.2", + "licenseId": "OLDAP-2.2.2", "seeAlso": [ - "https://paritylicense.com/versions/6.0.0.html" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003ddf2cc1e21eb7c160695f5b7cffd6296c151ba188" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.1-only.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-only.json", + "reference": "https://spdx.org/licenses/GPL-2.0-with-font-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-font-exception.json", "referenceNumber": 314, - "name": "GNU Lesser General Public License v2.1 only", - "licenseId": "LGPL-2.1-only", + "name": "GNU General Public License v2.0 w/Font exception", + "licenseId": "GPL-2.0-with-font-exception", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", - "https://opensource.org/licenses/LGPL-2.1" + "https://www.gnu.org/licenses/gpl-faq.html#FontException" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MPL-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0.json", "referenceNumber": 315, - "name": "Mozilla Public License 1.1", - "licenseId": "MPL-1.1", + "name": "Creative Commons Attribution 3.0 Unported", + "licenseId": "CC-BY-3.0", "seeAlso": [ - "http://www.mozilla.org/MPL/MPL-1.1.html", - "https://opensource.org/licenses/MPL-1.1" + "https://creativecommons.org/licenses/by/3.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/bzip2-1.0.5.html", + "reference": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.5.json", + "detailsUrl": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.json", "referenceNumber": 316, - "name": "bzip2 and libbzip2 License v1.0.5", - "licenseId": "bzip2-1.0.5", + "name": "PolyForm Noncommercial License 1.0.0", + "licenseId": "PolyForm-Noncommercial-1.0.0", "seeAlso": [ - "https://sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html", - "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" + "https://polyformproject.org/licenses/noncommercial/1.0.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Cube.html", + "reference": "https://spdx.org/licenses/MIT.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Cube.json", + "detailsUrl": "https://spdx.org/licenses/MIT.json", "referenceNumber": 317, - "name": "Cube License", - "licenseId": "Cube", + "name": "MIT License", + "licenseId": "MIT", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Cube" + "https://opensource.org/licenses/MIT" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SISSL.html", + "reference": "https://spdx.org/licenses/CNRI-Python.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SISSL.json", + "detailsUrl": "https://spdx.org/licenses/CNRI-Python.json", "referenceNumber": 318, - "name": "Sun Industry Standards Source License v1.1", - "licenseId": "SISSL", + "name": "CNRI Python License", + "licenseId": "CNRI-Python", "seeAlso": [ - "http://www.openoffice.org/licenses/sissl_license.html", - "https://opensource.org/licenses/SISSL" + "https://opensource.org/licenses/CNRI-Python" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/PostgreSQL.html", + "reference": "https://spdx.org/licenses/WTFPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PostgreSQL.json", + "detailsUrl": "https://spdx.org/licenses/WTFPL.json", "referenceNumber": 319, - "name": "PostgreSQL License", - "licenseId": "PostgreSQL", + "name": "Do What The F*ck You Want To Public License", + "licenseId": "WTFPL", "seeAlso": [ - "http://www.postgresql.org/about/licence", - "https://opensource.org/licenses/PostgreSQL" + "http://www.wtfpl.net/about/", + "http://sam.zoy.org/wtfpl/COPYING" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CERN-OHL-P-2.0.html", + "reference": "https://spdx.org/licenses/AFL-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-P-2.0.json", + "detailsUrl": "https://spdx.org/licenses/AFL-3.0.json", "referenceNumber": 320, - "name": "CERN Open Hardware Licence Version 2 - Permissive", - "licenseId": "CERN-OHL-P-2.0", + "name": "Academic Free License v3.0", + "licenseId": "AFL-3.0", "seeAlso": [ - "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + "http://www.rosenlaw.com/AFL3.0.htm", + "https://opensource.org/licenses/afl-3.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OSL-3.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-3.0.json", + "reference": "https://spdx.org/licenses/bzip2-1.0.5.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.5.json", "referenceNumber": 321, - "name": "Open Software License 3.0", - "licenseId": "OSL-3.0", + "name": "bzip2 and libbzip2 License v1.0.5", + "licenseId": "bzip2-1.0.5", "seeAlso": [ - "https://web.archive.org/web/20120101081418/http://rosenlaw.com:80/OSL3.0.htm", - "https://opensource.org/licenses/OSL-3.0" + "https://sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html", + "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPPL-1.2.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.2.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.json", "referenceNumber": 322, - "name": "LaTeX Project Public License v1.2", - "licenseId": "LPPL-1.2", + "name": "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + "licenseId": "CC-BY-NC-SA-1.0", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-2.txt" + "https://creativecommons.org/licenses/by-nc-sa/1.0/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/xpp.html", + "reference": "https://spdx.org/licenses/SHL-0.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/xpp.json", + "detailsUrl": "https://spdx.org/licenses/SHL-0.5.json", "referenceNumber": 323, - "name": "XPP License", - "licenseId": "xpp", + "name": "Solderpad Hardware License v0.5", + "licenseId": "SHL-0.5", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/xpp" + "https://solderpad.org/licenses/SHL-0.5/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.8.html", + "reference": "https://spdx.org/licenses/GD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.8.json", + "detailsUrl": "https://spdx.org/licenses/GD.json", "referenceNumber": 324, - "name": "Open LDAP Public License v2.8", - "licenseId": "OLDAP-2.8", + "name": "GD License", + "licenseId": "GD", "seeAlso": [ - "http://www.openldap.org/software/release/license.html" + "https://libgd.github.io/manuals/2.3.0/files/license-txt.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AML.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AML.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.json", "referenceNumber": 325, - "name": "Apple MIT License", - "licenseId": "AML", + "name": "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + "licenseId": "CC-BY-NC-ND-2.5", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Apple_MIT_License" + "https://creativecommons.org/licenses/by-nc-nd/2.5/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Motosoto.html", + "reference": "https://spdx.org/licenses/NTP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Motosoto.json", + "detailsUrl": "https://spdx.org/licenses/NTP.json", "referenceNumber": 326, - "name": "Motosoto License", - "licenseId": "Motosoto", + "name": "NTP License", + "licenseId": "NTP", "seeAlso": [ - "https://opensource.org/licenses/Motosoto" + "https://opensource.org/licenses/NTP" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Vim.html", + "reference": "https://spdx.org/licenses/CrystalStacker.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Vim.json", + "detailsUrl": "https://spdx.org/licenses/CrystalStacker.json", "referenceNumber": 327, - "name": "Vim License", - "licenseId": "Vim", + "name": "CrystalStacker License", + "licenseId": "CrystalStacker", "seeAlso": [ - "http://vimdoc.sourceforge.net/htmldoc/uganda.html" + "https://fedoraproject.org/wiki/Licensing:CrystalStacker?rd\u003dLicensing/CrystalStacker" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TCP-wrappers.html", + "reference": "https://spdx.org/licenses/Baekmuk.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TCP-wrappers.json", + "detailsUrl": "https://spdx.org/licenses/Baekmuk.json", "referenceNumber": 328, - "name": "TCP Wrappers License", - "licenseId": "TCP-wrappers", + "name": "Baekmuk License", + "licenseId": "Baekmuk", "seeAlso": [ - "http://rc.quest.com/topics/openssh/license.php#tcpwrappers" + "https://fedoraproject.org/wiki/Licensing:Baekmuk?rd\u003dLicensing/Baekmuk" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Sendmail-8.23.html", + "reference": "https://spdx.org/licenses/AML.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Sendmail-8.23.json", + "detailsUrl": "https://spdx.org/licenses/AML.json", "referenceNumber": 329, - "name": "Sendmail License 8.23", - "licenseId": "Sendmail-8.23", + "name": "Apple MIT License", + "licenseId": "AML", "seeAlso": [ - "https://www.proofpoint.com/sites/default/files/sendmail-license.pdf", - "https://web.archive.org/web/20181003101040/https://www.proofpoint.com/sites/default/files/sendmail-license.pdf" + "https://fedoraproject.org/wiki/Licensing/Apple_MIT_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Ruby.html", + "reference": "https://spdx.org/licenses/XSkat.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Ruby.json", + "detailsUrl": "https://spdx.org/licenses/XSkat.json", "referenceNumber": 330, - "name": "Ruby License", - "licenseId": "Ruby", + "name": "XSkat License", + "licenseId": "XSkat", "seeAlso": [ - "http://www.ruby-lang.org/en/LICENSE.txt" + "https://fedoraproject.org/wiki/Licensing/XSkat_License" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.html", + "reference": "https://spdx.org/licenses/OGTSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.json", + "detailsUrl": "https://spdx.org/licenses/OGTSL.json", "referenceNumber": 331, - "name": "GNU Free Documentation License v1.1 or later - invariants", - "licenseId": "GFDL-1.1-invariants-or-later", + "name": "Open Group Test Suite License", + "licenseId": "OGTSL", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "http://www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt", + "https://opensource.org/licenses/OGTSL" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-2.5-AU.html", + "reference": "https://spdx.org/licenses/Motosoto.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5-AU.json", + "detailsUrl": "https://spdx.org/licenses/Motosoto.json", "referenceNumber": 332, - "name": "Creative Commons Attribution 2.5 Australia", - "licenseId": "CC-BY-2.5-AU", + "name": "Motosoto License", + "licenseId": "Motosoto", "seeAlso": [ - "https://creativecommons.org/licenses/by/2.5/au/legalcode" + "https://opensource.org/licenses/Motosoto" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/NetCDF.html", + "reference": "https://spdx.org/licenses/PHP-3.01.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NetCDF.json", + "detailsUrl": "https://spdx.org/licenses/PHP-3.01.json", "referenceNumber": 333, - "name": "NetCDF license", - "licenseId": "NetCDF", + "name": "PHP License v3.01", + "licenseId": "PHP-3.01", "seeAlso": [ - "http://www.unidata.ucar.edu/software/netcdf/copyright.html" + "http://www.php.net/license/3_01.txt" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-1-Clause.html", + "reference": "https://spdx.org/licenses/etalab-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-1-Clause.json", + "detailsUrl": "https://spdx.org/licenses/etalab-2.0.json", "referenceNumber": 334, - "name": "BSD 1-Clause License", - "licenseId": "BSD-1-Clause", + "name": "Etalab Open License 2.0", + "licenseId": "etalab-2.0", "seeAlso": [ - "https://svnweb.freebsd.org/base/head/include/ifaddrs.h?revision\u003d326823" + "https://github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf", + "https://raw.githubusercontent.com/DISIC/politique-de-contribution-open-source/master/LICENSE" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Leptonica.html", + "reference": "https://spdx.org/licenses/MPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Leptonica.json", + "detailsUrl": "https://spdx.org/licenses/MPL-1.0.json", "referenceNumber": 335, - "name": "Leptonica License", - "licenseId": "Leptonica", + "name": "Mozilla Public License 1.0", + "licenseId": "MPL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Leptonica" + "http://www.mozilla.org/MPL/MPL-1.0.html", + "https://opensource.org/licenses/MPL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/AGPL-3.0-or-later.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-or-later.json", + "reference": "https://spdx.org/licenses/GPL-2.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0+.json", "referenceNumber": 336, - "name": "GNU Affero General Public License v3.0 or later", - "licenseId": "AGPL-3.0-or-later", + "name": "GNU General Public License v2.0 or later", + "licenseId": "GPL-2.0+", "seeAlso": [ - "https://www.gnu.org/licenses/agpl.txt", - "https://opensource.org/licenses/AGPL-3.0" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Beerware.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Beerware.json", + "reference": "https://spdx.org/licenses/StandardML-NJ.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/StandardML-NJ.json", "referenceNumber": 337, - "name": "Beerware License", - "licenseId": "Beerware", + "name": "Standard ML of New Jersey License", + "licenseId": "StandardML-NJ", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Beerware", - "https://people.freebsd.org/~phk/" + "http://www.smlnj.org//license.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/EUPL-1.2.html", + "reference": "https://spdx.org/licenses/Interbase-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUPL-1.2.json", + "detailsUrl": "https://spdx.org/licenses/Interbase-1.0.json", "referenceNumber": 338, - "name": "European Union Public License 1.2", - "licenseId": "EUPL-1.2", + "name": "Interbase Public License v1.0", + "licenseId": "Interbase-1.0", "seeAlso": [ - "https://joinup.ec.europa.eu/page/eupl-text-11-12", - "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl_v1.2_en.pdf", - "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/2020-03/EUPL-1.2%20EN.txt", - "https://joinup.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt", - "http://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri\u003dCELEX:32017D0863", - "https://opensource.org/licenses/EUPL-1.2" + "https://web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OSET-PL-2.1.html", + "reference": "https://spdx.org/licenses/NPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSET-PL-2.1.json", + "detailsUrl": "https://spdx.org/licenses/NPL-1.0.json", "referenceNumber": 339, - "name": "OSET Public License version 2.1", - "licenseId": "OSET-PL-2.1", + "name": "Netscape Public License v1.0", + "licenseId": "NPL-1.0", "seeAlso": [ - "http://www.osetfoundation.org/public-license", - "https://opensource.org/licenses/OPL-2.1" + "http://www.mozilla.org/MPL/NPL/1.0/" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-1.0.html", + "reference": "https://spdx.org/licenses/eGenix.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-1.0.json", + "detailsUrl": "https://spdx.org/licenses/eGenix.json", "referenceNumber": 340, - "name": "Creative Commons Attribution 1.0 Generic", - "licenseId": "CC-BY-1.0", + "name": "eGenix.com Public License 1.1.0", + "licenseId": "eGenix", "seeAlso": [ - "https://creativecommons.org/licenses/by/1.0/legalcode" + "http://www.egenix.com/products/eGenix.com-Public-License-1.1.0.pdf", + "https://fedoraproject.org/wiki/Licensing/eGenix.com_Public_License_1.1.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SISSL-1.2.html", + "reference": "https://spdx.org/licenses/NICTA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SISSL-1.2.json", + "detailsUrl": "https://spdx.org/licenses/NICTA-1.0.json", "referenceNumber": 341, - "name": "Sun Industry Standards Source License v1.2", - "licenseId": "SISSL-1.2", + "name": "NICTA Public Software License, Version 1.0", + "licenseId": "NICTA-1.0", "seeAlso": [ - "http://gridscheduler.sourceforge.net/Gridengine_SISSL_license.html" + "https://opensource.apple.com/source/mDNSResponder/mDNSResponder-320.10/mDNSPosix/nss_ReadMe.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-3.0.html", + "reference": "https://spdx.org/licenses/PSF-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0.json", + "detailsUrl": "https://spdx.org/licenses/PSF-2.0.json", "referenceNumber": 342, - "name": "Creative Commons Attribution Non Commercial 3.0 Unported", - "licenseId": "CC-BY-NC-3.0", + "name": "Python Software Foundation License 2.0", + "licenseId": "PSF-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/3.0/legalcode" + "https://opensource.org/licenses/Python-2.0" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0+.json", + "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-only.json", "referenceNumber": 343, - "name": "GNU General Public License v2.0 or later", - "licenseId": "GPL-2.0+", + "name": "GNU Free Documentation License v1.3 only - invariants", + "licenseId": "GFDL-1.3-invariants-only", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-1.4.html", + "reference": "https://spdx.org/licenses/OGL-UK-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.4.json", + "detailsUrl": "https://spdx.org/licenses/OGL-UK-3.0.json", "referenceNumber": 344, - "name": "Open LDAP Public License v1.4", - "licenseId": "OLDAP-1.4", + "name": "Open Government Licence v3.0", + "licenseId": "OGL-UK-3.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dc9f95c2f3f2ffb5e0ae55fe7388af75547660941" + "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/X11.html", + "reference": "https://spdx.org/licenses/dvipdfm.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/X11.json", + "detailsUrl": "https://spdx.org/licenses/dvipdfm.json", "referenceNumber": 345, - "name": "X11 License", - "licenseId": "X11", + "name": "dvipdfm License", + "licenseId": "dvipdfm", "seeAlso": [ - "http://www.xfree86.org/3.3.6/COPYRIGHT2.html#3" + "https://fedoraproject.org/wiki/Licensing/dvipdfm" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Intel.html", + "reference": "https://spdx.org/licenses/DRL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Intel.json", + "detailsUrl": "https://spdx.org/licenses/DRL-1.0.json", "referenceNumber": 346, - "name": "Intel Open Source License", - "licenseId": "Intel", + "name": "Detection Rule License 1.0", + "licenseId": "DRL-1.0", "seeAlso": [ - "https://opensource.org/licenses/Intel" + "https://github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDLA-Permissive-1.0.html", + "reference": "https://spdx.org/licenses/Ruby.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Ruby.json", "referenceNumber": 347, - "name": "Community Data License Agreement Permissive 1.0", - "licenseId": "CDLA-Permissive-1.0", + "name": "Ruby License", + "licenseId": "Ruby", "seeAlso": [ - "https://cdla.io/permissive-1-0" + "http://www.ruby-lang.org/en/LICENSE.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NIST-PD.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NIST-PD.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json", "referenceNumber": 348, - "name": "NIST Public Domain Notice", - "licenseId": "NIST-PD", + "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + "licenseId": "CC-BY-NC-ND-3.0-IGO", "seeAlso": [ - "https://github.com/tcheneau/simpleRPL/blob/e645e69e38dd4e3ccfeceb2db8cba05b7c2e0cd3/LICENSE.txt", - "https://github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md" + "https://creativecommons.org/licenses/by-nc-nd/3.0/igo/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LiLiQ-R-1.1.html", + "reference": "https://spdx.org/licenses/APAFML.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LiLiQ-R-1.1.json", + "detailsUrl": "https://spdx.org/licenses/APAFML.json", "referenceNumber": 349, - "name": "Licence Libre du Québec – Réciprocité version 1.1", - "licenseId": "LiLiQ-R-1.1", + "name": "Adobe Postscript AFM License", + "licenseId": "APAFML", "seeAlso": [ - "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-liliq-r-v1-1/", - "http://opensource.org/licenses/LiLiQ-R-1.1" + "https://fedoraproject.org/wiki/Licensing/AdobePostscriptAFM" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SMLNJ.html", + "reference": "https://spdx.org/licenses/CDLA-Permissive-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SMLNJ.json", + "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-2.0.json", "referenceNumber": 350, - "name": "Standard ML of New Jersey License", - "licenseId": "SMLNJ", + "name": "Community Data License Agreement Permissive 2.0", + "licenseId": "CDLA-Permissive-2.0", "seeAlso": [ - "https://www.smlnj.org/license.html" + "https://cdla.dev/permissive-2-0" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BUSL-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BUSL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.0.json", "referenceNumber": 351, - "name": "Business Source License 1.1", - "licenseId": "BUSL-1.1", + "name": "Creative Commons Attribution Non Commercial 2.0 Generic", + "licenseId": "CC-BY-NC-2.0", "seeAlso": [ - "https://mariadb.com/bsl11/" + "https://creativecommons.org/licenses/by-nc/2.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CDDL-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDDL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.json", "referenceNumber": 352, - "name": "Common Development and Distribution License 1.1", - "licenseId": "CDDL-1.1", + "name": "Creative Commons Attribution No Derivatives 3.0 Germany", + "licenseId": "CC-BY-ND-3.0-DE", "seeAlso": [ - "http://glassfish.java.net/public/CDDL+GPL_1_1.html", - "https://javaee.github.io/glassfish/LICENSE" + "https://creativecommons.org/licenses/by-nd/3.0/de/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-3.0.html", + "reference": "https://spdx.org/licenses/OFL-1.1-RFN.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0.json", + "detailsUrl": "https://spdx.org/licenses/OFL-1.1-RFN.json", "referenceNumber": 353, - "name": "Creative Commons Attribution Share Alike 3.0 Unported", - "licenseId": "CC-BY-SA-3.0", + "name": "SIL Open Font License 1.1 with Reserved Font Name", + "licenseId": "OFL-1.1-RFN", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/3.0/legalcode" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", + "https://opensource.org/licenses/OFL-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.html", + "reference": "https://spdx.org/licenses/Caldera.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.json", + "detailsUrl": "https://spdx.org/licenses/Caldera.json", "referenceNumber": 354, - "name": "Creative Commons Attribution Share Alike 2.1 Japan", - "licenseId": "CC-BY-SA-2.1-JP", + "name": "Caldera License", + "licenseId": "Caldera", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode" + "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Wsuipa.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Wsuipa.json", + "reference": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.json", "referenceNumber": 355, - "name": "Wsuipa License", - "licenseId": "Wsuipa", + "name": "GNU General Public License v3.0 w/GCC Runtime Library exception", + "licenseId": "GPL-3.0-with-GCC-exception", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Wsuipa" + "https://www.gnu.org/licenses/gcc-exception-3.1.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.html", + "reference": "https://spdx.org/licenses/LPL-1.02.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.json", + "detailsUrl": "https://spdx.org/licenses/LPL-1.02.json", "referenceNumber": 356, - "name": "Mozilla Public License 2.0 (no copyleft exception)", - "licenseId": "MPL-2.0-no-copyleft-exception", + "name": "Lucent Public License v1.02", + "licenseId": "LPL-1.02", "seeAlso": [ - "http://www.mozilla.org/MPL/2.0/", - "https://opensource.org/licenses/MPL-2.0" + "http://plan9.bell-labs.com/plan9/license.html", + "https://opensource.org/licenses/LPL-1.02" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Unicode-DFS-2015.html", + "reference": "https://spdx.org/licenses/Beerware.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2015.json", + "detailsUrl": "https://spdx.org/licenses/Beerware.json", "referenceNumber": 357, - "name": "Unicode License Agreement - Data Files and Software (2015)", - "licenseId": "Unicode-DFS-2015", + "name": "Beerware License", + "licenseId": "Beerware", "seeAlso": [ - "https://web.archive.org/web/20151224134844/http://unicode.org/copyright.html" + "https://fedoraproject.org/wiki/Licensing/Beerware", + "https://people.freebsd.org/~phk/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.json", + "reference": "https://spdx.org/licenses/OFL-1.1-no-RFN.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OFL-1.1-no-RFN.json", "referenceNumber": 358, - "name": "GNU General Public License v2.0 w/Classpath exception", - "licenseId": "GPL-2.0-with-classpath-exception", + "name": "SIL Open Font License 1.1 with no Reserved Font Name", + "licenseId": "OFL-1.1-no-RFN", "seeAlso": [ - "https://www.gnu.org/software/classpath/license.html" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", + "https://opensource.org/licenses/OFL-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.html", + "reference": "https://spdx.org/licenses/OLDAP-1.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.3.json", "referenceNumber": 359, - "name": "CNRI Python Open Source GPL Compatible License Agreement", - "licenseId": "CNRI-Python-GPL-Compatible", + "name": "Open LDAP Public License v1.3", + "licenseId": "OLDAP-1.3", "seeAlso": [ - "http://www.python.org/download/releases/1.6.1/download_win/" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003de5f8117f0ce088d0bd7a8e18ddf37eaa40eb09b1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.0.1.html", + "reference": "https://spdx.org/licenses/APL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.1.json", + "detailsUrl": "https://spdx.org/licenses/APL-1.0.json", "referenceNumber": 360, - "name": "Open LDAP Public License v2.0.1", - "licenseId": "OLDAP-2.0.1", + "name": "Adaptive Public License 1.0", + "licenseId": "APL-1.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db6d68acd14e51ca3aab4428bf26522aa74873f0e" + "https://opensource.org/licenses/APL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/psutils.html", + "reference": "https://spdx.org/licenses/Linux-man-pages-copyleft.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/psutils.json", + "detailsUrl": "https://spdx.org/licenses/Linux-man-pages-copyleft.json", "referenceNumber": 361, - "name": "psutils License", - "licenseId": "psutils", + "name": "Linux man-pages Copyleft", + "licenseId": "Linux-man-pages-copyleft", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/psutils" + "https://www.kernel.org/doc/man-pages/licenses.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/eCos-2.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/eCos-2.0.json", + "reference": "https://spdx.org/licenses/curl.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/curl.json", "referenceNumber": 362, - "name": "eCos license version 2.0", - "licenseId": "eCos-2.0", + "name": "curl License", + "licenseId": "curl", "seeAlso": [ - "https://www.gnu.org/licenses/ecos-license.html" + "https://github.com/bagder/curl/blob/master/COPYING" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-2.0.html", + "reference": "https://spdx.org/licenses/GPL-2.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-2.0.json", + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-or-later.json", "referenceNumber": 363, - "name": "Creative Commons Attribution 2.0 Generic", - "licenseId": "CC-BY-2.0", + "name": "GNU General Public License v2.0 or later", + "licenseId": "GPL-2.0-or-later", "seeAlso": [ - "https://creativecommons.org/licenses/by/2.0/legalcode" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Artistic-2.0.html", + "reference": "https://spdx.org/licenses/SISSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Artistic-2.0.json", + "detailsUrl": "https://spdx.org/licenses/SISSL.json", "referenceNumber": 364, - "name": "Artistic License 2.0", - "licenseId": "Artistic-2.0", + "name": "Sun Industry Standards Source License v1.1", + "licenseId": "SISSL", "seeAlso": [ - "http://www.perlfoundation.org/artistic_license_2_0", - "https://www.perlfoundation.org/artistic-license-20.html", - "https://opensource.org/licenses/artistic-license-2.0" + "http://www.openoffice.org/licenses/sissl_license.html", + "https://opensource.org/licenses/SISSL" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Artistic-1.0-cl8.html", + "reference": "https://spdx.org/licenses/CC-BY-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Artistic-1.0-cl8.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-4.0.json", "referenceNumber": 365, - "name": "Artistic License 1.0 w/clause 8", - "licenseId": "Artistic-1.0-cl8", + "name": "Creative Commons Attribution 4.0 International", + "licenseId": "CC-BY-4.0", "seeAlso": [ - "https://opensource.org/licenses/Artistic-1.0" + "https://creativecommons.org/licenses/by/4.0/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Aladdin.html", + "reference": "https://spdx.org/licenses/CECILL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Aladdin.json", + "detailsUrl": "https://spdx.org/licenses/CECILL-1.1.json", "referenceNumber": 366, - "name": "Aladdin Free Public License", - "licenseId": "Aladdin", + "name": "CeCILL Free Software License Agreement v1.1", + "licenseId": "CECILL-1.1", "seeAlso": [ - "http://pages.cs.wisc.edu/~ghost/doc/AFPL/6.01/Public.htm" + "http://www.cecill.info/licences/Licence_CeCILL_V1.1-US.html" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-3.0-or-later.html", + "reference": "https://spdx.org/licenses/OGL-UK-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-or-later.json", + "detailsUrl": "https://spdx.org/licenses/OGL-UK-1.0.json", "referenceNumber": 367, - "name": "GNU Lesser General Public License v3.0 or later", - "licenseId": "LGPL-3.0-or-later", + "name": "Open Government Licence v1.0", + "licenseId": "OGL-UK-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://opensource.org/licenses/LGPL-3.0" + "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/1/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SNIA.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SNIA.json", + "reference": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.json", "referenceNumber": 368, - "name": "SNIA Public License 1.1", - "licenseId": "SNIA", + "name": "GNU General Public License v2.0 w/Bison exception", + "licenseId": "GPL-2.0-with-bison-exception", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/SNIA_Public_License" + "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.html", + "reference": "https://spdx.org/licenses/OLDAP-2.7.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.7.json", "referenceNumber": 369, - "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", - "licenseId": "CC-BY-NC-SA-3.0-IGO", + "name": "Open LDAP Public License v2.7", + "licenseId": "OLDAP-2.7", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/3.0/igo/legalcode" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d47c2415c1df81556eeb39be6cad458ef87c534a2" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/AGPL-3.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/AGPL-3.0.json", + "reference": "https://spdx.org/licenses/Glulxe.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Glulxe.json", "referenceNumber": 370, - "name": "GNU Affero General Public License v3.0", - "licenseId": "AGPL-3.0", + "name": "Glulxe License", + "licenseId": "Glulxe", "seeAlso": [ - "https://www.gnu.org/licenses/agpl.txt", - "https://opensource.org/licenses/AGPL-3.0" + "https://fedoraproject.org/wiki/Licensing/Glulxe" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SSPL-1.0.html", + "reference": "https://spdx.org/licenses/BSD-4-Clause-UC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SSPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-UC.json", "referenceNumber": 371, - "name": "Server Side Public License, v 1", - "licenseId": "SSPL-1.0", + "name": "BSD-4-Clause (University of California-Specific)", + "licenseId": "BSD-4-Clause-UC", "seeAlso": [ - "https://www.mongodb.com/licensing/server-side-public-license" + "http://www.freebsd.org/copyright/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/RPSL-1.0.html", + "reference": "https://spdx.org/licenses/LPPL-1.3a.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RPSL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.3a.json", "referenceNumber": 372, - "name": "RealNetworks Public Source License v1.0", - "licenseId": "RPSL-1.0", + "name": "LaTeX Project Public License v1.3a", + "licenseId": "LPPL-1.3a", "seeAlso": [ - "https://helixcommunity.org/content/rpsl", - "https://opensource.org/licenses/RPSL-1.0" + "http://www.latex-project.org/lppl/lppl-1-3a.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MIT-open-group.html", + "reference": "https://spdx.org/licenses/Zlib.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-open-group.json", + "detailsUrl": "https://spdx.org/licenses/Zlib.json", "referenceNumber": 373, - "name": "MIT Open Group variant", - "licenseId": "MIT-open-group", + "name": "zlib License", + "licenseId": "Zlib", "seeAlso": [ - "https://gitlab.freedesktop.org/xorg/app/iceauth/-/blob/master/COPYING", - "https://gitlab.freedesktop.org/xorg/app/xvinfo/-/blob/master/COPYING", - "https://gitlab.freedesktop.org/xorg/app/xsetroot/-/blob/master/COPYING", - "https://gitlab.freedesktop.org/xorg/app/xauth/-/blob/master/COPYING" + "http://www.zlib.net/zlib_license.html", + "https://opensource.org/licenses/Zlib" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-LBNL.html", + "reference": "https://spdx.org/licenses/Crossword.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-LBNL.json", + "detailsUrl": "https://spdx.org/licenses/Crossword.json", "referenceNumber": 374, - "name": "Lawrence Berkeley National Labs BSD variant license", - "licenseId": "BSD-3-Clause-LBNL", + "name": "Crossword License", + "licenseId": "Crossword", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/LBNLBSD" + "https://fedoraproject.org/wiki/Licensing/Crossword" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.1.html", "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1.json", "referenceNumber": 375, - "name": "GNU General Public License v1.0 only", - "licenseId": "GPL-1.0", + "name": "GNU Free Documentation License v1.1", + "licenseId": "GFDL-1.1", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CECILL-B.html", + "reference": "https://spdx.org/licenses/BSD-4-Clause.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-B.json", + "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause.json", "referenceNumber": 376, - "name": "CeCILL-B Free Software License Agreement", - "licenseId": "CECILL-B", + "name": "BSD 4-Clause \"Original\" or \"Old\" License", + "licenseId": "BSD-4-Clause", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html" + "http://directory.fsf.org/wiki/License:BSD_4Clause" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Frameworx-1.0.html", + "reference": "https://spdx.org/licenses/Condor-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Frameworx-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Condor-1.1.json", "referenceNumber": 377, - "name": "Frameworx Open License 1.0", - "licenseId": "Frameworx-1.0", + "name": "Condor Public License v1.1", + "licenseId": "Condor-1.1", "seeAlso": [ - "https://opensource.org/licenses/Frameworx-1.0" + "http://research.cs.wisc.edu/condor/license.html#condor", + "http://web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/RHeCos-1.1.html", + "reference": "https://spdx.org/licenses/SSH-short.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RHeCos-1.1.json", + "detailsUrl": "https://spdx.org/licenses/SSH-short.json", "referenceNumber": 378, - "name": "Red Hat eCos Public License v1.1", - "licenseId": "RHeCos-1.1", + "name": "SSH short notice", + "licenseId": "SSH-short", "seeAlso": [ - "http://ecos.sourceware.org/old-license.html" + "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h", + "http://web.mit.edu/kolya/.f/root/athena.mit.edu/sipb.mit.edu/project/openssh/OldFiles/src/openssh-2.9.9p2/ssh-add.1", + "https://joinup.ec.europa.eu/svn/lesoll/trunk/italc/lib/src/dsa_key.cpp" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LAL-1.2.html", + "reference": "https://spdx.org/licenses/Borceux.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LAL-1.2.json", + "detailsUrl": "https://spdx.org/licenses/Borceux.json", "referenceNumber": 379, - "name": "Licence Art Libre 1.2", - "licenseId": "LAL-1.2", + "name": "Borceux license", + "licenseId": "Borceux", "seeAlso": [ - "http://artlibre.org/licence/lal/licence-art-libre-12/" + "https://fedoraproject.org/wiki/Licensing/Borceux" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OML.html", + "reference": "https://spdx.org/licenses/GPL-1.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OML.json", + "detailsUrl": "https://spdx.org/licenses/GPL-1.0-only.json", "referenceNumber": 380, - "name": "Open Market License", - "licenseId": "OML", + "name": "GNU General Public License v1.0 only", + "licenseId": "GPL-1.0-only", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Open_Market_License" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/UCL-1.0.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Clear.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/UCL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Clear.json", "referenceNumber": 381, - "name": "Upstream Compatibility License v1.0", - "licenseId": "UCL-1.0", + "name": "BSD 3-Clause Clear License", + "licenseId": "BSD-3-Clause-Clear", "seeAlso": [ - "https://opensource.org/licenses/UCL-1.0" + "http://labs.metacarta.com/license-explanation.html#license" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.json", + "reference": "https://spdx.org/licenses/FTL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/FTL.json", "referenceNumber": 382, - "name": "GNU General Public License v2.0 w/Autoconf exception", - "licenseId": "GPL-2.0-with-autoconf-exception", + "name": "Freetype Project License", + "licenseId": "FTL", "seeAlso": [ - "http://ac-archive.sourceforge.net/doc/copyright.html" + "http://freetype.fis.uniroma2.it/FTL.TXT", + "http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT", + "http://gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/APL-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.json", "referenceNumber": 383, - "name": "Adaptive Public License 1.0", - "licenseId": "APL-1.0", + "name": "Creative Commons Attribution Share Alike 3.0 Austria", + "licenseId": "CC-BY-SA-3.0-AT", "seeAlso": [ - "https://opensource.org/licenses/APL-1.0" + "https://creativecommons.org/licenses/by-sa/3.0/at/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.2.html", + "reference": "https://spdx.org/licenses/Hippocratic-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.json", + "detailsUrl": "https://spdx.org/licenses/Hippocratic-2.1.json", "referenceNumber": 384, - "name": "Open LDAP Public License v2.2", - "licenseId": "OLDAP-2.2", + "name": "Hippocratic License 2.1", + "licenseId": "Hippocratic-2.1", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d470b0c18ec67621c85881b2733057fecf4a1acc3" + "https://firstdonoharm.dev/version/2/1/license.html", + "https://github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CERN-OHL-W-2.0.html", + "reference": "https://spdx.org/licenses/Spencer-99.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-W-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Spencer-99.json", "referenceNumber": 385, - "name": "CERN Open Hardware Licence Version 2 - Weakly Reciprocal", - "licenseId": "CERN-OHL-W-2.0", + "name": "Spencer License 99", + "licenseId": "Spencer-99", "seeAlso": [ - "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + "http://www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/DRL-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/DRL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-only.json", "referenceNumber": 386, - "name": "Detection Rule License 1.0", - "licenseId": "DRL-1.0", + "name": "GNU Free Documentation License v1.2 only", + "licenseId": "GFDL-1.2-only", "seeAlso": [ - "https://github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-3.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0+.json", + "reference": "https://spdx.org/licenses/Intel-ACPI.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Intel-ACPI.json", "referenceNumber": 387, - "name": "GNU General Public License v3.0 or later", - "licenseId": "GPL-3.0+", + "name": "Intel ACPI Software License Agreement", + "licenseId": "Intel-ACPI", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "https://fedoraproject.org/wiki/Licensing/Intel_ACPI_Software_License_Agreement" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-1.0.html", + "reference": "https://spdx.org/licenses/AFL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-1.0.json", + "detailsUrl": "https://spdx.org/licenses/AFL-2.0.json", "referenceNumber": 388, - "name": "Creative Commons Attribution No Derivatives 1.0 Generic", - "licenseId": "CC-BY-ND-1.0", + "name": "Academic Free License v2.0", + "licenseId": "AFL-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/1.0/legalcode" + "http://wayback.archive.org/web/20060924134533/http://www.opensource.org/licenses/afl-2.0.txt" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.json", "referenceNumber": 389, - "name": "GNU Free Documentation License v1.3 or later - no invariants", - "licenseId": "GFDL-1.3-no-invariants-or-later", + "name": "BSD 3-Clause No Military License", + "licenseId": "BSD-3-Clause-No-Military-License", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://gitlab.syncad.com/hive/dhive/-/blob/master/LICENSE", + "https://github.com/greymass/swift-eosio/blob/master/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CECILL-1.1.html", + "reference": "https://spdx.org/licenses/Libpng.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/Libpng.json", "referenceNumber": 390, - "name": "CeCILL Free Software License Agreement v1.1", - "licenseId": "CECILL-1.1", + "name": "libpng License", + "licenseId": "Libpng", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V1.1-US.html" + "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Noweb.html", + "reference": "https://spdx.org/licenses/OFL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Noweb.json", + "detailsUrl": "https://spdx.org/licenses/OFL-1.0.json", "referenceNumber": 391, - "name": "Noweb License", - "licenseId": "Noweb", + "name": "SIL Open Font License 1.0", + "licenseId": "OFL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Noweb" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MakeIndex.html", + "reference": "https://spdx.org/licenses/MTLL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MakeIndex.json", + "detailsUrl": "https://spdx.org/licenses/MTLL.json", "referenceNumber": 392, - "name": "MakeIndex License", - "licenseId": "MakeIndex", + "name": "Matrix Template Library License", + "licenseId": "MTLL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MakeIndex" + "https://fedoraproject.org/wiki/Licensing/Matrix_Template_Library_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MS-RL.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MS-RL.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json", "referenceNumber": 393, - "name": "Microsoft Reciprocal License", - "licenseId": "MS-RL", + "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + "licenseId": "CC-BY-NC-ND-3.0-DE", "seeAlso": [ - "http://www.microsoft.com/opensource/licenses.mspx", - "https://opensource.org/licenses/MS-RL" + "https://creativecommons.org/licenses/by-nc-nd/3.0/de/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TORQUE-1.1.html", + "reference": "https://spdx.org/licenses/EFL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TORQUE-1.1.json", + "detailsUrl": "https://spdx.org/licenses/EFL-1.0.json", "referenceNumber": 394, - "name": "TORQUE v2.5+ Software License v1.1", - "licenseId": "TORQUE-1.1", + "name": "Eiffel Forum License v1.0", + "licenseId": "EFL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/TORQUEv1.1" + "http://www.eiffel-nice.org/license/forum.txt", + "https://opensource.org/licenses/EFL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/AFL-3.0.html", + "reference": "https://spdx.org/licenses/FDK-AAC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-3.0.json", + "detailsUrl": "https://spdx.org/licenses/FDK-AAC.json", "referenceNumber": 395, - "name": "Academic Free License v3.0", - "licenseId": "AFL-3.0", + "name": "Fraunhofer FDK AAC Codec Library", + "licenseId": "FDK-AAC", "seeAlso": [ - "http://www.rosenlaw.com/AFL3.0.htm", - "https://opensource.org/licenses/afl-3.0" + "https://fedoraproject.org/wiki/Licensing/FDK-AAC", + "https://directory.fsf.org/wiki/License:Fdk" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-only.html", + "reference": "https://spdx.org/licenses/CECILL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-only.json", + "detailsUrl": "https://spdx.org/licenses/CECILL-1.0.json", "referenceNumber": 396, - "name": "GNU Free Documentation License v1.1 only - invariants", - "licenseId": "GFDL-1.1-invariants-only", + "name": "CeCILL Free Software License Agreement v1.0", + "licenseId": "CECILL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "http://www.cecill.info/licences/Licence_CeCILL_V1-fr.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1.json", + "reference": "https://spdx.org/licenses/PHP-3.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/PHP-3.0.json", "referenceNumber": 397, - "name": "GNU Free Documentation License v1.1", - "licenseId": "GFDL-1.1", + "name": "PHP License v3.0", + "licenseId": "PHP-3.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "http://www.php.net/license/3_0.txt", + "https://opensource.org/licenses/PHP-3.0" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/FTL.html", + "reference": "https://spdx.org/licenses/FreeImage.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FTL.json", + "detailsUrl": "https://spdx.org/licenses/FreeImage.json", "referenceNumber": 398, - "name": "Freetype Project License", - "licenseId": "FTL", + "name": "FreeImage Public License v1.0", + "licenseId": "FreeImage", "seeAlso": [ - "http://freetype.fis.uniroma2.it/FTL.TXT", - "http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT", - "http://gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT" + "http://freeimage.sourceforge.net/freeimage-license.txt" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AFL-2.0.html", + "reference": "https://spdx.org/licenses/iMatix.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/iMatix.json", "referenceNumber": 399, - "name": "Academic Free License v2.0", - "licenseId": "AFL-2.0", + "name": "iMatix Standard Function Library Agreement", + "licenseId": "iMatix", "seeAlso": [ - "http://wayback.archive.org/web/20060924134533/http://www.opensource.org/licenses/afl-2.0.txt" + "http://legacy.imatix.com/html/sfl/sfl4.htm#license" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.json", + "reference": "https://spdx.org/licenses/TOSL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/TOSL.json", "referenceNumber": 400, - "name": "GNU General Public License v3.0 w/Autoconf exception", - "licenseId": "GPL-3.0-with-autoconf-exception", + "name": "Trusster Open Source License", + "licenseId": "TOSL", "seeAlso": [ - "https://www.gnu.org/licenses/autoconf-exception-3.0.html" + "https://fedoraproject.org/wiki/Licensing/TOSL" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPPL-1.3a.html", + "reference": "https://spdx.org/licenses/MIT-CMU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.3a.json", + "detailsUrl": "https://spdx.org/licenses/MIT-CMU.json", "referenceNumber": 401, - "name": "LaTeX Project Public License v1.3a", - "licenseId": "LPPL-1.3a", + "name": "CMU License", + "licenseId": "MIT-CMU", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-3a.txt" + "https://fedoraproject.org/wiki/Licensing:MIT?rd\u003dLicensing/MIT#CMU_Style", + "https://github.com/python-pillow/Pillow/blob/fffb426092c8db24a5f4b6df243a8a3c01fb63cd/LICENSE" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-1.2.html", + "reference": "https://spdx.org/licenses/CATOSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.2.json", + "detailsUrl": "https://spdx.org/licenses/CATOSL-1.1.json", "referenceNumber": 402, - "name": "Open LDAP Public License v1.2", - "licenseId": "OLDAP-1.2", + "name": "Computer Associates Trusted Open Source License 1.1", + "licenseId": "CATOSL-1.1", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d42b0383c50c299977b5893ee695cf4e486fb0dc7" + "https://opensource.org/licenses/CATOSL-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/ODbL-1.0.html", + "reference": "https://spdx.org/licenses/LPPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ODbL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.1.json", "referenceNumber": 403, - "name": "Open Data Commons Open Database License v1.0", - "licenseId": "ODbL-1.0", + "name": "LaTeX Project Public License v1.1", + "licenseId": "LPPL-1.1", "seeAlso": [ - "http://www.opendatacommons.org/licenses/odbl/1.0/", - "https://opendatacommons.org/licenses/odbl/1-0/" + "http://www.latex-project.org/lppl/lppl-1-1.txt" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json", "referenceNumber": 404, - "name": "Licence Libre du Québec – Réciprocité forte version 1.1", - "licenseId": "LiLiQ-Rplus-1.1", + "name": "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + "licenseId": "CC-BY-NC-SA-2.0-FR", "seeAlso": [ - "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-forte-liliq-r-v1-1/", - "http://opensource.org/licenses/LiLiQ-Rplus-1.1" + "https://creativecommons.org/licenses/by-nc-sa/2.0/fr/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Qhull.html", + "reference": "https://spdx.org/licenses/LGPL-2.1-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Qhull.json", + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-or-later.json", "referenceNumber": 405, - "name": "Qhull License", - "licenseId": "Qhull", + "name": "GNU Lesser General Public License v2.1 or later", + "licenseId": "LGPL-2.1-or-later", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Qhull" + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OCCT-PL.html", + "reference": "https://spdx.org/licenses/Arphic-1999.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OCCT-PL.json", + "detailsUrl": "https://spdx.org/licenses/Arphic-1999.json", "referenceNumber": 406, - "name": "Open CASCADE Technology Public License", - "licenseId": "OCCT-PL", + "name": "Arphic Public License", + "licenseId": "Arphic-1999", "seeAlso": [ - "http://www.opencascade.com/content/occt-public-license" + "http://ftp.gnu.org/gnu/non-gnu/chinese-fonts-truetype/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NTP.html", + "reference": "https://spdx.org/licenses/Sendmail-8.23.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NTP.json", + "detailsUrl": "https://spdx.org/licenses/Sendmail-8.23.json", "referenceNumber": 407, - "name": "NTP License", - "licenseId": "NTP", + "name": "Sendmail License 8.23", + "licenseId": "Sendmail-8.23", "seeAlso": [ - "https://opensource.org/licenses/NTP" + "https://www.proofpoint.com/sites/default/files/sendmail-license.pdf", + "https://web.archive.org/web/20181003101040/https://www.proofpoint.com/sites/default/files/sendmail-license.pdf" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EUPL-1.1.html", + "reference": "https://spdx.org/licenses/CECILL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUPL-1.1.json", + "detailsUrl": "https://spdx.org/licenses/CECILL-2.0.json", "referenceNumber": 408, - "name": "European Union Public License 1.1", - "licenseId": "EUPL-1.1", + "name": "CeCILL Free Software License Agreement v2.0", + "licenseId": "CECILL-2.0", "seeAlso": [ - "https://joinup.ec.europa.eu/software/page/eupl/licence-eupl", - "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl1.1.-licence-en_0.pdf", - "https://opensource.org/licenses/EUPL-1.1" + "http://www.cecill.info/licences/Licence_CeCILL_V2-en.html" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CPOL-1.02.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CPOL-1.02.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.json", "referenceNumber": 409, - "name": "Code Project Open License 1.02", - "licenseId": "CPOL-1.02", + "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + "licenseId": "CC-BY-NC-ND-3.0", "seeAlso": [ - "http://www.codeproject.com/info/cpol10.aspx" + "https://creativecommons.org/licenses/by-nc-nd/3.0/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OCLC-2.0.html", + "reference": "https://spdx.org/licenses/APSL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OCLC-2.0.json", + "detailsUrl": "https://spdx.org/licenses/APSL-2.0.json", "referenceNumber": 410, - "name": "OCLC Research Public License 2.0", - "licenseId": "OCLC-2.0", + "name": "Apple Public Source License 2.0", + "licenseId": "APSL-2.0", "seeAlso": [ - "http://www.oclc.org/research/activities/software/license/v2final.htm", - "https://opensource.org/licenses/OCLC-2.0" + "http://www.opensource.apple.com/license/apsl/" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.1.html", + "reference": "https://spdx.org/licenses/Unicode-TOU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.1.json", + "detailsUrl": "https://spdx.org/licenses/Unicode-TOU.json", "referenceNumber": 411, - "name": "Open LDAP Public License v2.1", - "licenseId": "OLDAP-2.1", + "name": "Unicode Terms of Use", + "licenseId": "Unicode-TOU", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db0d176738e96a0d3b9f85cb51e140a86f21be715" + "http://www.unicode.org/copyright.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Intel-ACPI.html", + "reference": "https://spdx.org/licenses/MS-RL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Intel-ACPI.json", + "detailsUrl": "https://spdx.org/licenses/MS-RL.json", "referenceNumber": 412, - "name": "Intel ACPI Software License Agreement", - "licenseId": "Intel-ACPI", + "name": "Microsoft Reciprocal License", + "licenseId": "MS-RL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Intel_ACPI_Software_License_Agreement" + "http://www.microsoft.com/opensource/licenses.mspx", + "https://opensource.org/licenses/MS-RL" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OFL-1.0-RFN.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.0-RFN.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.json", "referenceNumber": 413, - "name": "SIL Open Font License 1.0 with Reserved Font Name", - "licenseId": "OFL-1.0-RFN", + "name": "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + "licenseId": "CC-BY-NC-ND-2.0", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" + "https://creativecommons.org/licenses/by-nc-nd/2.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC0-1.0.html", + "reference": "https://spdx.org/licenses/OSL-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC0-1.0.json", + "detailsUrl": "https://spdx.org/licenses/OSL-3.0.json", "referenceNumber": 414, - "name": "Creative Commons Zero v1.0 Universal", - "licenseId": "CC0-1.0", + "name": "Open Software License 3.0", + "licenseId": "OSL-3.0", "seeAlso": [ - "https://creativecommons.org/publicdomain/zero/1.0/legalcode" + "https://web.archive.org/web/20120101081418/http://rosenlaw.com:80/OSL3.0.htm", + "https://opensource.org/licenses/OSL-3.0" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LGPL-2.0-or-later.html", + "reference": "https://spdx.org/licenses/Nokia.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-or-later.json", + "detailsUrl": "https://spdx.org/licenses/Nokia.json", "referenceNumber": 415, - "name": "GNU Library General Public License v2 or later", - "licenseId": "LGPL-2.0-or-later", + "name": "Nokia Open Source License", + "licenseId": "Nokia", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + "https://opensource.org/licenses/nokia" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SGI-B-1.1.html", + "reference": "https://spdx.org/licenses/MPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SGI-B-1.1.json", + "detailsUrl": "https://spdx.org/licenses/MPL-1.1.json", "referenceNumber": 416, - "name": "SGI Free Software License B v1.1", - "licenseId": "SGI-B-1.1", + "name": "Mozilla Public License 1.1", + "licenseId": "MPL-1.1", "seeAlso": [ - "http://oss.sgi.com/projects/FreeB/" + "http://www.mozilla.org/MPL/MPL-1.1.html", + "https://opensource.org/licenses/MPL-1.1" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.json", + "reference": "https://spdx.org/licenses/GPL-1.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-1.0.json", "referenceNumber": 417, - "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", - "licenseId": "CC-BY-NC-ND-3.0", + "name": "GNU General Public License v1.0 only", + "licenseId": "GPL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/3.0/legalcode" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/etalab-2.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/etalab-2.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-1.0.json", "referenceNumber": 418, - "name": "Etalab Open License 2.0", - "licenseId": "etalab-2.0", + "name": "Creative Commons Attribution Non Commercial 1.0 Generic", + "licenseId": "CC-BY-NC-1.0", "seeAlso": [ - "https://github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf", - "https://raw.githubusercontent.com/DISIC/politique-de-contribution-open-source/master/LICENSE" + "https://creativecommons.org/licenses/by-nc/1.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.html", + "reference": "https://spdx.org/licenses/CERN-OHL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.1.json", "referenceNumber": 419, - "name": "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", - "licenseId": "CC-BY-NC-SA-2.0-FR", + "name": "CERN Open Hardware Licence v1.1", + "licenseId": "CERN-OHL-1.1", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.0/fr/legalcode" + "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0-only.html", + "reference": "https://spdx.org/licenses/W3C-20150513.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-only.json", + "detailsUrl": "https://spdx.org/licenses/W3C-20150513.json", "referenceNumber": 420, - "name": "GNU General Public License v3.0 only", - "licenseId": "GPL-3.0-only", + "name": "W3C Software Notice and Document License (2015-05-13)", + "licenseId": "W3C-20150513", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "https://www.w3.org/Consortium/Legal/2015/copyright-software-and-document" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Zend-2.0.html", + "reference": "https://spdx.org/licenses/SPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zend-2.0.json", + "detailsUrl": "https://spdx.org/licenses/SPL-1.0.json", "referenceNumber": 421, - "name": "Zend License v2.0", - "licenseId": "Zend-2.0", + "name": "Sun Public License v1.0", + "licenseId": "SPL-1.0", "seeAlso": [ - "https://web.archive.org/web/20130517195954/http://www.zend.com/license/2_00.txt" + "https://opensource.org/licenses/SPL-1.0" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MirOS.html", + "reference": "https://spdx.org/licenses/LZMA-SDK-9.22.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MirOS.json", + "detailsUrl": "https://spdx.org/licenses/LZMA-SDK-9.22.json", "referenceNumber": 422, - "name": "The MirOS Licence", - "licenseId": "MirOS", + "name": "LZMA SDK License (versions 9.22 and beyond)", + "licenseId": "LZMA-SDK-9.22", "seeAlso": [ - "https://opensource.org/licenses/MirOS" + "https://www.7-zip.org/sdk.html", + "https://sourceforge.net/projects/sevenzip/files/LZMA%20SDK/" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.html", + "reference": "https://spdx.org/licenses/LGPLLR.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.json", + "detailsUrl": "https://spdx.org/licenses/LGPLLR.json", "referenceNumber": 423, - "name": "BSD 3-Clause No Nuclear License 2014", - "licenseId": "BSD-3-Clause-No-Nuclear-License-2014", + "name": "Lesser General Public License For Linguistic Resources", + "licenseId": "LGPLLR", "seeAlso": [ - "https://java.net/projects/javaeetutorial/pages/BerkeleyLicense" + "http://www-igm.univ-mlv.fr/~unitex/lgpllr.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BitTorrent-1.0.html", + "reference": "https://spdx.org/licenses/LiLiQ-R-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.0.json", + "detailsUrl": "https://spdx.org/licenses/LiLiQ-R-1.1.json", "referenceNumber": 424, - "name": "BitTorrent Open Source License v1.0", - "licenseId": "BitTorrent-1.0", + "name": "Licence Libre du Québec – Réciprocité version 1.1", + "licenseId": "LiLiQ-R-1.1", "seeAlso": [ - "http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1\u003d1.1\u0026r2\u003d1.1.1.1\u0026diff_format\u003ds" + "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-liliq-r-v1-1/", + "http://opensource.org/licenses/LiLiQ-R-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-3.0.html", + "reference": "https://spdx.org/licenses/ZPL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0.json", + "detailsUrl": "https://spdx.org/licenses/ZPL-2.0.json", "referenceNumber": 425, - "name": "Creative Commons Attribution No Derivatives 3.0 Unported", - "licenseId": "CC-BY-ND-3.0", + "name": "Zope Public License 2.0", + "licenseId": "ZPL-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/3.0/legalcode" + "http://old.zope.org/Resources/License/ZPL-2.0", + "https://opensource.org/licenses/ZPL-2.0" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MIT.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT.json", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.json", "referenceNumber": 426, - "name": "MIT License", - "licenseId": "MIT", + "name": "BSD 3-Clause No Nuclear Warranty", + "licenseId": "BSD-3-Clause-No-Nuclear-Warranty", "seeAlso": [ - "https://opensource.org/licenses/MIT" + "https://jogamp.org/git/?p\u003dgluegen.git;a\u003dblob_plain;f\u003dLICENSE.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NIST-PD-fallback.html", + "reference": "https://spdx.org/licenses/Spencer-94.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NIST-PD-fallback.json", + "detailsUrl": "https://spdx.org/licenses/Spencer-94.json", "referenceNumber": 427, - "name": "NIST Public Domain Notice with license fallback", - "licenseId": "NIST-PD-fallback", + "name": "Spencer License 94", + "licenseId": "Spencer-94", "seeAlso": [ - "https://github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE", - "https://github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst" + "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.5.json", "referenceNumber": 428, - "name": "Creative Commons Attribution Non Commercial 3.0 Germany", - "licenseId": "CC-BY-NC-3.0-DE", + "name": "Creative Commons Attribution Share Alike 2.5 Generic", + "licenseId": "CC-BY-SA-2.5", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/3.0/de/legalcode" + "https://creativecommons.org/licenses/by-sa/2.5/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FSFULLR.html", + "reference": "https://spdx.org/licenses/OSET-PL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FSFULLR.json", + "detailsUrl": "https://spdx.org/licenses/OSET-PL-2.1.json", "referenceNumber": 429, - "name": "FSF Unlimited License (with License Retention)", - "licenseId": "FSFULLR", + "name": "OSET Public License version 2.1", + "licenseId": "OSET-PL-2.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant" + "http://www.osetfoundation.org/public-license", + "https://opensource.org/licenses/OPL-2.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-1.0.html", + "reference": "https://spdx.org/licenses/IPA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-1.0.json", + "detailsUrl": "https://spdx.org/licenses/IPA.json", "referenceNumber": 430, - "name": "Creative Commons Attribution Non Commercial 1.0 Generic", - "licenseId": "CC-BY-NC-1.0", + "name": "IPA Font License", + "licenseId": "IPA", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/1.0/legalcode" + "https://opensource.org/licenses/IPA" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Python-2.0.html", + "reference": "https://spdx.org/licenses/Glide.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Python-2.0.json", + "detailsUrl": "https://spdx.org/licenses/Glide.json", "referenceNumber": 431, - "name": "Python License 2.0", - "licenseId": "Python-2.0", + "name": "3dfx Glide License", + "licenseId": "Glide", "seeAlso": [ - "https://opensource.org/licenses/Python-2.0" + "http://www.users.on.net/~triforce/glidexp/COPYING.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ANTLR-PD-fallback.html", + "reference": "https://spdx.org/licenses/RSA-MD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ANTLR-PD-fallback.json", + "detailsUrl": "https://spdx.org/licenses/RSA-MD.json", "referenceNumber": 432, - "name": "ANTLR Software Rights Notice with license fallback", - "licenseId": "ANTLR-PD-fallback", + "name": "RSA Message-Digest License", + "licenseId": "RSA-MD", "seeAlso": [ - "http://www.antlr2.org/license.html" + "http://www.faqs.org/rfcs/rfc1321.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MulanPSL-1.0.html", + "reference": "https://spdx.org/licenses/CPAL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MulanPSL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/CPAL-1.0.json", "referenceNumber": 433, - "name": "Mulan Permissive Software License, Version 1", - "licenseId": "MulanPSL-1.0", + "name": "Common Public Attribution License 1.0", + "licenseId": "CPAL-1.0", "seeAlso": [ - "https://license.coscl.org.cn/MulanPSL/", - "https://github.com/yuwenlong/longphp/blob/25dfb70cc2a466dc4bb55ba30901cbce08d164b5/LICENSE" + "https://opensource.org/licenses/CPAL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/wxWindows.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/wxWindows.json", + "reference": "https://spdx.org/licenses/UPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/UPL-1.0.json", "referenceNumber": 434, - "name": "wxWindows Library License", - "licenseId": "wxWindows", + "name": "Universal Permissive License v1.0", + "licenseId": "UPL-1.0", "seeAlso": [ - "https://opensource.org/licenses/WXwindows" + "https://opensource.org/licenses/UPL" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-4.0.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-4.0.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.0.json", "referenceNumber": 435, - "name": "Creative Commons Attribution 4.0 International", - "licenseId": "CC-BY-4.0", + "name": "Creative Commons Attribution No Derivatives 2.0 Generic", + "licenseId": "CC-BY-ND-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by/4.0/legalcode" + "https://creativecommons.org/licenses/by-nd/2.0/legalcode" ], "isOsiApproved": false, - "isFsfLibre": true + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/Rdisc.html", + "reference": "https://spdx.org/licenses/libtiff.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Rdisc.json", + "detailsUrl": "https://spdx.org/licenses/libtiff.json", "referenceNumber": 436, - "name": "Rdisc License", - "licenseId": "Rdisc", + "name": "libtiff License", + "licenseId": "libtiff", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Rdisc_License" + "https://fedoraproject.org/wiki/Licensing/libtiff" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MIT-enna.html", + "reference": "https://spdx.org/licenses/zlib-acknowledgement.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-enna.json", + "detailsUrl": "https://spdx.org/licenses/zlib-acknowledgement.json", "referenceNumber": 437, - "name": "enna License", - "licenseId": "MIT-enna", + "name": "zlib/libpng License with Acknowledgement", + "licenseId": "zlib-acknowledgement", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT#enna" + "https://fedoraproject.org/wiki/Licensing/ZlibWithAcknowledgement" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-4-Clause-UC.html", + "reference": "https://spdx.org/licenses/BSD-Source-Code.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-UC.json", + "detailsUrl": "https://spdx.org/licenses/BSD-Source-Code.json", "referenceNumber": 438, - "name": "BSD-4-Clause (University of California-Specific)", - "licenseId": "BSD-4-Clause-UC", + "name": "BSD Source Code Attribution", + "licenseId": "BSD-Source-Code", "seeAlso": [ - "http://www.freebsd.org/copyright/license.html" + "https://github.com/robbiehanson/CocoaHTTPServer/blob/master/LICENSE.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MPL-2.0.html", + "reference": "https://spdx.org/licenses/BitTorrent-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.0.json", "referenceNumber": 439, - "name": "Mozilla Public License 2.0", - "licenseId": "MPL-2.0", + "name": "BitTorrent Open Source License v1.0", + "licenseId": "BitTorrent-1.0", "seeAlso": [ - "https://www.mozilla.org/MPL/2.0/", - "https://opensource.org/licenses/MPL-2.0" + "http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1\u003d1.1\u0026r2\u003d1.1.1.1\u0026diff_format\u003ds" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/YPL-1.0.html", + "reference": "https://spdx.org/licenses/HPND-sell-variant.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/YPL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/HPND-sell-variant.json", "referenceNumber": 440, - "name": "Yahoo! Public License v1.0", - "licenseId": "YPL-1.0", + "name": "Historical Permission Notice and Disclaimer - sell variant", + "licenseId": "HPND-sell-variant", "seeAlso": [ - "http://www.zimbra.com/license/yahoo_public_license_1.0.html" + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h\u003dv4.19" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ZPL-2.0.html", + "reference": "https://spdx.org/licenses/W3C-19980720.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ZPL-2.0.json", + "detailsUrl": "https://spdx.org/licenses/W3C-19980720.json", "referenceNumber": 441, - "name": "Zope Public License 2.0", - "licenseId": "ZPL-2.0", + "name": "W3C Software Notice and License (1998-07-20)", + "licenseId": "W3C-19980720", "seeAlso": [ - "http://old.zope.org/Resources/License/ZPL-2.0", - "https://opensource.org/licenses/ZPL-2.0" + "http://www.w3.org/Consortium/Legal/copyright-software-19980720.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/copyleft-next-0.3.1.html", + "reference": "https://spdx.org/licenses/CERN-OHL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.1.json", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.2.json", "referenceNumber": 442, - "name": "copyleft-next 0.3.1", - "licenseId": "copyleft-next-0.3.1", + "name": "CERN Open Hardware Licence v1.2", + "licenseId": "CERN-OHL-1.2", "seeAlso": [ - "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.1" + "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.html", + "reference": "https://spdx.org/licenses/OFL-1.0-RFN.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.json", + "detailsUrl": "https://spdx.org/licenses/OFL-1.0-RFN.json", "referenceNumber": 443, - "name": "GNU Free Documentation License v1.3 or later - invariants", - "licenseId": "GFDL-1.3-invariants-or-later", + "name": "SIL Open Font License 1.0 with Reserved Font Name", + "licenseId": "OFL-1.0-RFN", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPL-1.02.html", + "reference": "https://spdx.org/licenses/CC-PDDC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPL-1.02.json", + "detailsUrl": "https://spdx.org/licenses/CC-PDDC.json", "referenceNumber": 444, - "name": "Lucent Public License v1.02", - "licenseId": "LPL-1.02", + "name": "Creative Commons Public Domain Dedication and Certification", + "licenseId": "CC-PDDC", "seeAlso": [ - "http://plan9.bell-labs.com/plan9/license.html", - "https://opensource.org/licenses/LPL-1.02" + "https://creativecommons.org/licenses/publicdomain/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NLOD-1.0.html", + "reference": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NLOD-1.0.json", + "detailsUrl": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.json", "referenceNumber": 445, - "name": "Norwegian Licence for Open Government Data (NLOD) 1.0", - "licenseId": "NLOD-1.0", + "name": "PolyForm Small Business License 1.0.0", + "licenseId": "PolyForm-Small-Business-1.0.0", "seeAlso": [ - "http://data.norge.no/nlod/en/1.0" + "https://polyformproject.org/licenses/small-business/1.0.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MIT-Modern-Variant.html", + "reference": "https://spdx.org/licenses/Watcom-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-Modern-Variant.json", + "detailsUrl": "https://spdx.org/licenses/Watcom-1.0.json", "referenceNumber": 446, - "name": "MIT License Modern Variant", - "licenseId": "MIT-Modern-Variant", + "name": "Sybase Open Watcom Public License 1.0", + "licenseId": "Watcom-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:MIT#Modern_Variants", - "https://ptolemy.berkeley.edu/copyright.htm", - "https://pirlwww.lpl.arizona.edu/resources/guide/software/PerlTk/Tixlic.html" + "https://opensource.org/licenses/Watcom-1.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/MTLL.html", + "reference": "https://spdx.org/licenses/CC-BY-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MTLL.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.0.json", "referenceNumber": 447, - "name": "Matrix Template Library License", - "licenseId": "MTLL", + "name": "Creative Commons Attribution 2.0 Generic", + "licenseId": "CC-BY-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Matrix_Template_Library_License" + "https://creativecommons.org/licenses/by/2.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ECL-1.0.html", + "reference": "https://spdx.org/licenses/RPL-1.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ECL-1.0.json", + "detailsUrl": "https://spdx.org/licenses/RPL-1.5.json", "referenceNumber": 448, - "name": "Educational Community License v1.0", - "licenseId": "ECL-1.0", + "name": "Reciprocal Public License 1.5", + "licenseId": "RPL-1.5", "seeAlso": [ - "https://opensource.org/licenses/ECL-1.0" + "https://opensource.org/licenses/RPL-1.5" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/EPICS.html", + "reference": "https://spdx.org/licenses/NLOD-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EPICS.json", + "detailsUrl": "https://spdx.org/licenses/NLOD-2.0.json", "referenceNumber": 449, - "name": "EPICS Open License", - "licenseId": "EPICS", + "name": "Norwegian Licence for Open Government Data (NLOD) 2.0", + "licenseId": "NLOD-2.0", "seeAlso": [ - "https://epics.anl.gov/license/open.php" + "http://data.norge.no/nlod/en/2.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Attribution.html", + "reference": "https://spdx.org/licenses/copyleft-next-0.3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Attribution.json", + "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.0.json", "referenceNumber": 450, - "name": "BSD with attribution", - "licenseId": "BSD-3-Clause-Attribution", + "name": "copyleft-next 0.3.0", + "licenseId": "copyleft-next-0.3.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/BSD_with_Attribution" + "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-only.html", + "reference": "https://spdx.org/licenses/RPSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-only.json", + "detailsUrl": "https://spdx.org/licenses/RPSL-1.0.json", "referenceNumber": 451, - "name": "GNU Free Documentation License v1.2 only - invariants", - "licenseId": "GFDL-1.2-invariants-only", + "name": "RealNetworks Public Source License v1.0", + "licenseId": "RPSL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://helixcommunity.org/content/rpsl", + "https://opensource.org/licenses/RPSL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/AMDPLPA.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AMDPLPA.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-1.0.json", "referenceNumber": 452, - "name": "AMD\u0027s plpa_map.c License", - "licenseId": "AMDPLPA", + "name": "Creative Commons Attribution No Derivatives 1.0 Generic", + "licenseId": "CC-BY-ND-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AMD_plpa_map_License" + "https://creativecommons.org/licenses/by-nd/1.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/FSFUL.html", + "reference": "https://spdx.org/licenses/LiLiQ-P-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FSFUL.json", + "detailsUrl": "https://spdx.org/licenses/LiLiQ-P-1.1.json", "referenceNumber": 453, - "name": "FSF Unlimited License", - "licenseId": "FSFUL", + "name": "Licence Libre du Québec – Permissive version 1.1", + "licenseId": "LiLiQ-P-1.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License" + "https://forge.gouv.qc.ca/licence/fr/liliq-v1-1/", + "http://opensource.org/licenses/LiLiQ-P-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-AT.html", + "reference": "https://spdx.org/licenses/FreeBSD-DOC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-AT.json", + "detailsUrl": "https://spdx.org/licenses/FreeBSD-DOC.json", "referenceNumber": 454, - "name": "Creative Commons Attribution 3.0 Austria", - "licenseId": "CC-BY-3.0-AT", + "name": "FreeBSD Documentation License", + "licenseId": "FreeBSD-DOC", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/at/legalcode" + "https://www.freebsd.org/copyright/freebsd-doc-license/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDLA-Sharing-1.0.html", + "reference": "https://spdx.org/licenses/Afmparse.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDLA-Sharing-1.0.json", + "detailsUrl": "https://spdx.org/licenses/Afmparse.json", "referenceNumber": 455, - "name": "Community Data License Agreement Sharing 1.0", - "licenseId": "CDLA-Sharing-1.0", + "name": "Afmparse License", + "licenseId": "Afmparse", "seeAlso": [ - "https://cdla.io/sharing-1-0" + "https://fedoraproject.org/wiki/Licensing/Afmparse" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.6.html", + "reference": "https://spdx.org/licenses/VSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.6.json", + "detailsUrl": "https://spdx.org/licenses/VSL-1.0.json", "referenceNumber": 456, - "name": "Open LDAP Public License v2.6", - "licenseId": "OLDAP-2.6", + "name": "Vovida Software License v1.0", + "licenseId": "VSL-1.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d1cae062821881f41b73012ba816434897abf4205" + "https://opensource.org/licenses/VSL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/LAL-1.3.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LAL-1.3.json", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.json", "referenceNumber": 457, - "name": "Licence Art Libre 1.3", - "licenseId": "LAL-1.3", + "name": "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + "licenseId": "CC-BY-NC-SA-2.5", "seeAlso": [ - "https://artlibre.org/" + "https://creativecommons.org/licenses/by-nc-sa/2.5/legalcode" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Jam.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Jam.json", + "referenceNumber": 458, + "name": "Jam License", + "licenseId": "Jam", + "seeAlso": [ + "https://www.boost.org/doc/libs/1_35_0/doc/html/jam.html", + "https://web.archive.org/web/20160330173339/https://swarm.workshop.perforce.com/files/guest/perforce_software/jam/src/README" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/OML.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OML.json", + "referenceNumber": 459, + "name": "Open Market License", + "licenseId": "OML", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Open_Market_License" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/ADSL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ADSL.json", + "referenceNumber": 460, + "name": "Amazon Digital Services License", + "licenseId": "ADSL", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/AmazonDigitalServicesLicense" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Zed.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Zed.json", + "referenceNumber": 461, + "name": "Zed License", + "licenseId": "Zed", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Zed" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Bitstream-Vera.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Bitstream-Vera.json", + "referenceNumber": 462, + "name": "Bitstream Vera Font License", + "licenseId": "Bitstream-Vera", + "seeAlso": [ + "https://web.archive.org/web/20080207013128/http://www.gnome.org/fonts/", + "https://docubrain.com/sites/default/files/licenses/bitstream-vera.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/IPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/IPL-1.0.json", + "referenceNumber": 463, + "name": "IBM Public License v1.0", + "licenseId": "IPL-1.0", + "seeAlso": [ + "https://opensource.org/licenses/IPL-1.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/BSL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSL-1.0.json", + "referenceNumber": 464, + "name": "Boost Software License 1.0", + "licenseId": "BSL-1.0", + "seeAlso": [ + "http://www.boost.org/LICENSE_1_0.txt", + "https://opensource.org/licenses/BSL-1.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/MirOS.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MirOS.json", + "referenceNumber": 465, + "name": "The MirOS Licence", + "licenseId": "MirOS", + "seeAlso": [ + "https://opensource.org/licenses/MirOS" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/W3C.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/W3C.json", + "referenceNumber": 466, + "name": "W3C Software Notice and License (2002-12-31)", + "licenseId": "W3C", + "seeAlso": [ + "http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231.html", + "https://opensource.org/licenses/W3C" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-only.json", + "referenceNumber": 467, + "name": "GNU Free Documentation License v1.2 only - invariants", + "licenseId": "GFDL-1.2-invariants-only", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/VOSTROM.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/VOSTROM.json", + "referenceNumber": 468, + "name": "VOSTROM Public License for Open Source", + "licenseId": "VOSTROM", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/VOSTROM" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/CERN-OHL-S-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-S-2.0.json", + "referenceNumber": 469, + "name": "CERN Open Hardware Licence Version 2 - Strongly Reciprocal", + "licenseId": "CERN-OHL-S-2.0", + "seeAlso": [ + "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/Apache-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Apache-1.1.json", + "referenceNumber": 470, + "name": "Apache License 1.1", + "licenseId": "Apache-1.1", + "seeAlso": [ + "http://apache.org/licenses/LICENSE-1.1", + "https://opensource.org/licenses/Apache-1.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/NASA-1.3.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NASA-1.3.json", + "referenceNumber": 471, + "name": "NASA Open Source Agreement 1.3", + "licenseId": "NASA-1.3", + "seeAlso": [ + "http://ti.arc.nasa.gov/opensource/nosa/", + "https://opensource.org/licenses/NASA-1.3" + ], + "isOsiApproved": true, + "isFsfLibre": false + }, + { + "reference": "https://spdx.org/licenses/SHL-0.51.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/SHL-0.51.json", + "referenceNumber": 472, + "name": "Solderpad Hardware License, Version 0.51", + "licenseId": "SHL-0.51", + "seeAlso": [ + "https://solderpad.org/licenses/SHL-0.51/" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OPUBL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OPUBL-1.0.json", + "referenceNumber": 473, + "name": "Open Publication License v1.0", + "licenseId": "OPUBL-1.0", + "seeAlso": [ + "http://opencontent.org/openpub/", + "https://www.debian.org/opl", + "https://www.ctan.org/license/opl" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/OCLC-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OCLC-2.0.json", + "referenceNumber": 474, + "name": "OCLC Research Public License 2.0", + "licenseId": "OCLC-2.0", + "seeAlso": [ + "http://www.oclc.org/research/activities/software/license/v2final.htm", + "https://opensource.org/licenses/OCLC-2.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OPUBL-1.0.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OPUBL-1.0.json", - "referenceNumber": 458, - "name": "Open Publication License v1.0", - "licenseId": "OPUBL-1.0", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.json", + "referenceNumber": 475, + "name": "BSD 3-Clause Open MPI variant", + "licenseId": "BSD-3-Clause-Open-MPI", "seeAlso": [ - "http://opencontent.org/openpub/", - "https://www.debian.org/opl", - "https://www.ctan.org/license/opl" + "https://www.open-mpi.org/community/license.php", + "http://www.netlib.org/lapack/LICENSE.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Bahyph.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Bahyph.json", - "referenceNumber": 459, - "name": "Bahyph License", - "licenseId": "Bahyph", + "reference": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.json", + "referenceNumber": 476, + "name": "GNU General Public License v2.0 w/GCC Runtime Library exception", + "licenseId": "GPL-2.0-with-GCC-exception", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Bahyph" + "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.html", + "reference": "https://spdx.org/licenses/Fair.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.json", - "referenceNumber": 460, - "name": "Creative Commons Attribution Share Alike 3.0 Austria", - "licenseId": "CC-BY-SA-3.0-AT", + "detailsUrl": "https://spdx.org/licenses/Fair.json", + "referenceNumber": 477, + "name": "Fair License", + "licenseId": "Fair", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/3.0/at/legalcode" + "http://fairlicense.org/", + "https://opensource.org/licenses/Fair" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/AFL-2.1.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-2.1.json", - "referenceNumber": 461, - "name": "Academic Free License v2.1", - "licenseId": "AFL-2.1", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.json", + "referenceNumber": 478, + "name": "GNU Free Documentation License v1.1 or later - no invariants", + "licenseId": "GFDL-1.1-no-invariants-or-later", "seeAlso": [ - "http://opensource.linux-mirror.org/licenses/afl-2.1.txt" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NTP-0.html", + "reference": "https://spdx.org/licenses/Saxpath.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NTP-0.json", - "referenceNumber": 462, - "name": "NTP No Attribution", - "licenseId": "NTP-0", + "detailsUrl": "https://spdx.org/licenses/Saxpath.json", + "referenceNumber": 479, + "name": "Saxpath License", + "licenseId": "Saxpath", "seeAlso": [ - "https://github.com/tytso/e2fsprogs/blob/master/lib/et/et_name.c" + "https://fedoraproject.org/wiki/Licensing/Saxpath_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OSL-2.1.html", + "reference": "https://spdx.org/licenses/SimPL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-2.1.json", - "referenceNumber": 463, - "name": "Open Software License 2.1", - "licenseId": "OSL-2.1", + "detailsUrl": "https://spdx.org/licenses/SimPL-2.0.json", + "referenceNumber": 480, + "name": "Simple Public License 2.0", + "licenseId": "SimPL-2.0", "seeAlso": [ - "http://web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm", - "https://opensource.org/licenses/OSL-2.1" + "https://opensource.org/licenses/SimPL-2.0" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/LGPL-2.0.html", + "reference": "https://spdx.org/licenses/LGPL-2.0+.html", "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0.json", - "referenceNumber": 464, - "name": "GNU Library General Public License v2 only", - "licenseId": "LGPL-2.0", + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0+.json", + "referenceNumber": 481, + "name": "GNU Library General Public License v2 or later", + "licenseId": "LGPL-2.0+", "seeAlso": [ "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/NCGL-UK-2.0.html", + "reference": "https://spdx.org/licenses/OLDAP-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NCGL-UK-2.0.json", - "referenceNumber": 465, - "name": "Non-Commercial Government Licence", - "licenseId": "NCGL-UK-2.0", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.5.json", + "referenceNumber": 482, + "name": "Open LDAP Public License v2.5", + "licenseId": "OLDAP-2.5", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/non-commercial-government-licence/version/2/" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d6852b9d90022e8593c98205413380536b1b5a7cf" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Glulxe.html", + "reference": "https://spdx.org/licenses/MIT-feh.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Glulxe.json", - "referenceNumber": 466, - "name": "Glulxe License", - "licenseId": "Glulxe", + "detailsUrl": "https://spdx.org/licenses/MIT-feh.json", + "referenceNumber": 483, + "name": "feh License", + "licenseId": "MIT-feh", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Glulxe" + "https://fedoraproject.org/wiki/Licensing/MIT#feh" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/copyleft-next-0.3.0.html", + "reference": "https://spdx.org/licenses/Barr.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.0.json", - "referenceNumber": 467, - "name": "copyleft-next 0.3.0", - "licenseId": "copyleft-next-0.3.0", + "detailsUrl": "https://spdx.org/licenses/Barr.json", + "referenceNumber": 484, + "name": "Barr License", + "licenseId": "Barr", "seeAlso": [ - "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.0" + "https://fedoraproject.org/wiki/Licensing/Barr" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/dvipdfm.html", + "reference": "https://spdx.org/licenses/Multics.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/dvipdfm.json", - "referenceNumber": 468, - "name": "dvipdfm License", - "licenseId": "dvipdfm", + "detailsUrl": "https://spdx.org/licenses/Multics.json", + "referenceNumber": 485, + "name": "Multics License", + "licenseId": "Multics", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/dvipdfm" + "https://opensource.org/licenses/Multics" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/libtiff.html", + "reference": "https://spdx.org/licenses/Adobe-Glyph.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/libtiff.json", - "referenceNumber": 469, - "name": "libtiff License", - "licenseId": "libtiff", + "detailsUrl": "https://spdx.org/licenses/Adobe-Glyph.json", + "referenceNumber": 486, + "name": "Adobe Glyph List License", + "licenseId": "Adobe-Glyph", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/libtiff" + "https://fedoraproject.org/wiki/Licensing/MIT#AdobeGlyph" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GLWTPL.html", + "reference": "https://spdx.org/licenses/TORQUE-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GLWTPL.json", - "referenceNumber": 470, - "name": "Good Luck With That Public License", - "licenseId": "GLWTPL", + "detailsUrl": "https://spdx.org/licenses/TORQUE-1.1.json", + "referenceNumber": 487, + "name": "TORQUE v2.5+ Software License v1.1", + "licenseId": "TORQUE-1.1", "seeAlso": [ - "https://github.com/me-shaon/GLWTPL/commit/da5f6bc734095efbacb442c0b31e33a65b9d6e85" + "https://fedoraproject.org/wiki/Licensing/TORQUEv1.1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/StandardML-NJ.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/StandardML-NJ.json", - "referenceNumber": 471, - "name": "Standard ML of New Jersey License", - "licenseId": "StandardML-NJ", + "reference": "https://spdx.org/licenses/BSD-2-Clause.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause.json", + "referenceNumber": 488, + "name": "BSD 2-Clause \"Simplified\" License", + "licenseId": "BSD-2-Clause", "seeAlso": [ - "http://www.smlnj.org//license.html" + "https://opensource.org/licenses/BSD-2-Clause" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CAL-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CAL-1.0.json", - "referenceNumber": 472, - "name": "Cryptographic Autonomy License 1.0", - "licenseId": "CAL-1.0", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-only.json", + "referenceNumber": 489, + "name": "GNU Free Documentation License v1.3 only", + "licenseId": "GFDL-1.3-only", "seeAlso": [ - "http://cryptographicautonomylicense.com/license-text.html", - "https://opensource.org/licenses/CAL-1.0" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { "reference": "https://spdx.org/licenses/Artistic-1.0-Perl.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/Artistic-1.0-Perl.json", - "referenceNumber": 473, + "referenceNumber": 490, "name": "Artistic License 1.0 (Perl)", "licenseId": "Artistic-1.0-Perl", "seeAlso": [ @@ -5960,38 +6173,41 @@ "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/gSOAP-1.3b.html", + "reference": "https://spdx.org/licenses/BSD-2-Clause-Views.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/gSOAP-1.3b.json", - "referenceNumber": 474, - "name": "gSOAP Public License v1.3b", - "licenseId": "gSOAP-1.3b", + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Views.json", + "referenceNumber": 491, + "name": "BSD 2-Clause with views sentence", + "licenseId": "BSD-2-Clause-Views", "seeAlso": [ - "http://www.cs.fsu.edu/~engelen/license.html" + "http://www.freebsd.org/copyright/freebsd-license.html", + "https://people.freebsd.org/~ivoras/wine/patch-wine-nvidia.sh", + "https://github.com/protegeproject/protege/blob/master/license.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.5.html", + "reference": "https://spdx.org/licenses/NPOSL-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.5.json", - "referenceNumber": 475, - "name": "Open LDAP Public License v2.5", - "licenseId": "OLDAP-2.5", + "detailsUrl": "https://spdx.org/licenses/NPOSL-3.0.json", + "referenceNumber": 492, + "name": "Non-Profit Open Software License 3.0", + "licenseId": "NPOSL-3.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d6852b9d90022e8593c98205413380536b1b5a7cf" + "https://opensource.org/licenses/NOSL3.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Interbase-1.0.html", + "reference": "https://spdx.org/licenses/Minpack.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Interbase-1.0.json", - "referenceNumber": 476, - "name": "Interbase Public License v1.0", - "licenseId": "Interbase-1.0", + "detailsUrl": "https://spdx.org/licenses/Minpack.json", + "referenceNumber": 493, + "name": "Minpack License", + "licenseId": "Minpack", "seeAlso": [ - "https://web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html" + "http://www.netlib.org/minpack/disclaimer", + "https://gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK" ], "isOsiApproved": false }, @@ -5999,7 +6215,7 @@ "reference": "https://spdx.org/licenses/LGPL-2.1.html", "isDeprecatedLicenseId": true, "detailsUrl": "https://spdx.org/licenses/LGPL-2.1.json", - "referenceNumber": 477, + "referenceNumber": 494, "name": "GNU Lesser General Public License v2.1 only", "licenseId": "LGPL-2.1", "seeAlso": [ @@ -6010,19 +6226,33 @@ "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MS-PL.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MS-PL.json", - "referenceNumber": 478, - "name": "Microsoft Public License", - "licenseId": "MS-PL", + "reference": "https://spdx.org/licenses/LGPL-3.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0.json", + "referenceNumber": 495, + "name": "GNU Lesser General Public License v3.0 only", + "licenseId": "LGPL-3.0", "seeAlso": [ - "http://www.microsoft.com/opensource/licenses.mspx", - "https://opensource.org/licenses/MS-PL" + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" ], "isOsiApproved": true, "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/CAL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CAL-1.0.json", + "referenceNumber": 496, + "name": "Cryptographic Autonomy License 1.0", + "licenseId": "CAL-1.0", + "seeAlso": [ + "http://cryptographicautonomylicense.com/license-text.html", + "https://opensource.org/licenses/CAL-1.0" + ], + "isOsiApproved": true } ], - "releaseDate": "2021-11-19" + "releaseDate": "2022-10-07" } \ No newline at end of file From 4fff51db46e0f6f14930a101d080f3defa59ccff Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 12 Oct 2022 09:40:51 +0100 Subject: [PATCH 019/489] added license reference block to SPDX --- src/scanoss/spdxlite.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 75e2a524..f9869f58 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -186,22 +186,28 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: }, 'documentNamespace': f'https://spdx.org/spdxdocs/scanoss-py-{__version__}-{md5hex}', 'documentDescribes': [], + 'hasExtractedLicensingInfos': [], 'packages': [] } + lic_refs = set() # Hash Set of non-SPDX license references for purl in raw_data: comp = raw_data.get(purl) - lic_names = [] licenses = comp.get('licenses') lic_text = 'NOASSERTION' if licenses: + lic_set = set() for lic in licenses: lc_id = lic.get('id') if lc_id: spdx_id = self.get_spdx_license_id(lc_id) - lic_names.append(spdx_id if spdx_id else lc_id) - if len(lic_names) > 0: - lic_text = ' AND '.join(lic_names) - if len(lic_names) > 1: + if not spdx_id: + if not lc_id.startswith('LicenseRef'): + lc_id = f'LicenseRef-{lc_id}' # Make sure it has a license ref in its name + lic_refs.add(lc_id) # save non-SPDX license for later reference + lic_set.add(spdx_id if spdx_id else lc_id) + if len(lic_set) > 0: + lic_text = ' AND '.join(lic_set) + if len(lic_set) > 1: lic_text = f'({lic_text})' # wrap the names in () if there is more than one comp_name = comp.get('component') comp_ver = comp.get('version') @@ -225,7 +231,24 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'referenceType': 'purl' }] }) - # End for loop + # End purls for loop + for lic_ref in lic_refs: # Insert all the non-SPDX license references + source = '' + match = re.search(r'^LicenseRef-(scancode-|scanoss-|)(\S+)$', lic_ref, re.IGNORECASE) + if match: + source = match.group(1).replace('-', '') # source for the custom license + name = match.group(2) # license name (without references, etc.) + else: + name = lic_ref + name = name.replace('-', ' ') + source = f' by {source}.' if source else '.' + data['hasExtractedLicensingInfos'].append({ + 'licenseId': lic_ref, + 'name': name, + 'extractedText': 'Detected license, please review component source code.', + 'comment': f'Detected license{source}' + }) + # End license refs for loop file = sys.stdout if not output_file and self.output_file: output_file = self.output_file @@ -258,6 +281,8 @@ def load_license_data(self) -> None: Load the embedded SPDX valid license JSON files Parse its contents to provide a lookup for valid name """ + # SPDX license files details from: https://spdx.org/licenses/ + # Specifically the JSON files come from GitHub: https://github.com/spdx/license-list-data/tree/master/json self._spdx_licenses = {} self._spdx_lic_names = {} self.print_debug('Loading SPDX License details...') @@ -319,7 +344,7 @@ def get_spdx_license_id(self, lic_name: str) -> str: lic_id = self._spdx_lic_names.get(search_name_dashes) if lic_id: return lic_id - self.print_stderr(f'Warning: Failed to find valid SPDX license identifier for: {lic_name}') + self.print_debug(f'Warning: Failed to find valid SPDX license identifier for: {lic_name}') return None # # End of SpdxLite Class From 95e439791a643f3346f59c0ce2b7869f845a7937 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 12 Oct 2022 09:41:42 +0100 Subject: [PATCH 020/489] fixed SPDX and CDX output data, added request id to gRPC --- CHANGELOG.md | 8 ++++++++ src/scanoss/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b364fb0..8b603d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.1.0] - 2022-10-12 +### Fixed +- Added LicenseRef info to SPDX Lite output +- Updated CycloneDX output format to support version 1.4 +### Added +- Added request id to gRPC requests + ## [1.0.6] - 2022-09-19 ### Added - Added support for scancode 2.0 output format @@ -109,3 +116,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.0.0]: https://github.com/scanoss/scanoss.py/compare/v0.9.0...v1.0.0 [1.0.4]: https://github.com/scanoss/scanoss.py/compare/v1.0.0...v1.0.4 [1.0.5]: https://github.com/scanoss/scanoss.py/compare/v1.0.4...v1.0.6 +[1.0.6]: https://github.com/scanoss/scanoss.py/compare/v1.0.6...v1.1.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 065cc485..c44e7627 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.0.6' +__version__ = '1.1.0' From b97f2cca691086e6714ae70a8ed9767d6b30a394 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 19 Oct 2022 10:37:20 +0100 Subject: [PATCH 021/489] fixed yarn.lock file dependency analysis issue --- CHANGELOG.md | 9 +++++++-- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 2 +- src/scanoss/scancodedeps.py | 17 +++++++++-------- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b603d9b..287c391e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.1.1] - 2022-10-19 +### Fixed +- Fixed issue with dependency parsing of yarn.lock files + ## [1.1.0] - 2022-10-12 ### Fixed - Added LicenseRef info to SPDX Lite output @@ -115,5 +119,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [0.9.0]: https://github.com/scanoss/scanoss.py/compare/v0.7.4...v0.9.0 [1.0.0]: https://github.com/scanoss/scanoss.py/compare/v0.9.0...v1.0.0 [1.0.4]: https://github.com/scanoss/scanoss.py/compare/v1.0.0...v1.0.4 -[1.0.5]: https://github.com/scanoss/scanoss.py/compare/v1.0.4...v1.0.6 -[1.0.6]: https://github.com/scanoss/scanoss.py/compare/v1.0.6...v1.1.0 +[1.0.6]: https://github.com/scanoss/scanoss.py/compare/v1.0.4...v1.0.6 +[1.1.0]: https://github.com/scanoss/scanoss.py/compare/v1.0.6...v1.1.0 +[1.1.1]: https://github.com/scanoss/scanoss.py/compare/v1.1.0...v1.1.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index c44e7627..198a16b9 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.1.0' +__version__ = '1.1.1' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index e217684d..6bb6d1a9 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -109,7 +109,7 @@ def setup_args() -> None: # Sub-command: dependency p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], description=f'Produce dependency file summary: {__version__}', - help='Scan source code for dependencies') + help='Scan source code for dependencies, but do not decorate them') p_dep.set_defaults(func=dependency) p_dep.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_dep.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index 2a4ca0c2..abd0b94c 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -103,28 +103,29 @@ def produce_from_json(self, data: json) -> dict: f_packages = fd.get('packages') # scancode formate 1.0 if not f_packages or f_packages == '': continue - # print(f'Path: {f_path}, Packages: {f_packages}') + self.print_debug(f'Path: {f_path}, Packages: {len(f_packages)}') + purls = [] for pkgs in f_packages: pk_deps = pkgs.get('dependencies') if not pk_deps or pk_deps == '': continue - # print(f'Path: {f_path}, Deps: {pk_deps}') - purls = [] + self.print_debug(f'Path: {f_path}, Dependencies: {len(pk_deps)}') for d in pk_deps: dp = d.get('purl') if not dp or dp == '': continue + dp = dp.replace('"', '').replace('%22', '') # remove unwanted quotes on purls dp_data = {'purl': dp} rq = d.get('extracted_requirement') # scancode format 2.0 if not rq or rq == '': rq = d.get('requirement') # scancode format 1.0 - if rq and rq != '' and not dp.endswith(rq): + # skip requirement if it ends with the purl (i.e. exact version) or if it's local (file) + if rq and rq != '' and not dp.endswith(rq) and not rq.startswith('file:'): dp_data['requirement'] = rq purls.append(dp_data) - # print(f'Path: {f_path}, Purls: {purls}') - if len(purls) > 0: - file = {'file': f_path, 'purls': purls} - files.append(file) + # self.print_stderr(f'Path: {f_path}, Purls: {purls}') + if len(purls) > 0: + files.append({'file': f_path, 'purls': purls}) # End packages # End file details # End dependencies json From 8b744b6bda0e4e28c8257d723c6fa7baa6896daf Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 5 Nov 2022 17:00:01 +0000 Subject: [PATCH 022/489] updating scancode requirements --- README.md | 5 ++--- requirements-scancode.txt | 2 ++ tests/data/requirements.txt | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 requirements-scancode.txt diff --git a/README.md b/README.md index 1c4a9e9d..4fea90ab 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,9 @@ pip3 install -r requirements.txt pip3 install -r requirements-dev.txt ``` -To enable dependency scanning an extra tool is required: scancode-toolkit +To enable dependency scanning, an extra tool is required: scancode-toolkit ```bash -pip3 install typecode-libmagic -pip3 install scancode-toolkit-mini +pip3 install -r requirements-scancode.txt ``` ### Package Development diff --git a/requirements-scancode.txt b/requirements-scancode.txt new file mode 100644 index 00000000..930b9b07 --- /dev/null +++ b/requirements-scancode.txt @@ -0,0 +1,2 @@ +typecode-libmagic +scancode-toolkit-mini diff --git a/tests/data/requirements.txt b/tests/data/requirements.txt index bee304b6..db3b752f 100644 --- a/tests/data/requirements.txt +++ b/tests/data/requirements.txt @@ -2,3 +2,5 @@ requests crc32c>=2.2 binaryornot progress +grpcio<=1.42.0 +protobuf>=3.16.0,<=3.19.1 From 2b31413eac587c2901726f4892058613ed855a72 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 5 Nov 2022 17:00:23 +0000 Subject: [PATCH 023/489] added base vulnerabilities gRPC api --- src/scanoss/api/vulnerabilities/__init__.py | 23 + .../api/vulnerabilities/v2/__init__.py | 23 + .../v2/scanoss_vulnerabilities_pb2.py | 449 ++++++++++++++++++ .../v2/scanoss_vulnerabilities_pb2_grpc.py | 142 ++++++ tests/csvoutput-test.py | 48 ++ 5 files changed, 685 insertions(+) create mode 100644 src/scanoss/api/vulnerabilities/__init__.py create mode 100644 src/scanoss/api/vulnerabilities/v2/__init__.py create mode 100644 src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py create mode 100644 src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py create mode 100644 tests/csvoutput-test.py diff --git a/src/scanoss/api/vulnerabilities/__init__.py b/src/scanoss/api/vulnerabilities/__init__.py new file mode 100644 index 00000000..0ac8eded --- /dev/null +++ b/src/scanoss/api/vulnerabilities/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/vulnerabilities/v2/__init__.py b/src/scanoss/api/vulnerabilities/v2/__init__.py new file mode 100644 index 00000000..0ac8eded --- /dev/null +++ b/src/scanoss/api/vulnerabilities/v2/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py new file mode 100644 index 00000000..208e778c --- /dev/null +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -0,0 +1,449 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto', + package='scanoss.api.vulnerabilities.v2', + syntax='proto3', + serialized_options=b'Z?github.com/scanoss/papi/api/vulnerabilitiesv2;vulnerabilitiesv2', + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n Date: Wed, 9 Nov 2022 12:02:58 +0000 Subject: [PATCH 024/489] Added vulnerability reporting --- src/scanoss/cyclonedx.py | 90 +++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 7f4a420d..56e01fb4 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -48,13 +48,14 @@ def parse(self, data: json): """ Parse the given input (raw/plain) JSON string and return CycloneDX summary :param data: json - JSON object - :return: CycloneDX dictionary + :return: CycloneDX dictionary, and vulnerability dictionary """ if not data: self.print_stderr('ERROR: No JSON data provided to parse.') - return None + return None, None self.print_debug(f'Processing raw results into CycloneDX format...') cdx = {} + vdx = {} for f in data: file_details = data.get(f) # print(f'File: {f}: {file_details}') @@ -101,10 +102,36 @@ def parse(self, data: json): if not purl: self.print_stderr(f'Warning: No PURL found for {f}: {file_details}') continue + fd = {} + vulnerabilities = d.get("vulnerabilities") + if vulnerabilities: + for vuln in vulnerabilities: + vuln_id = vuln.get("ID") + if vuln_id == '': + vuln_id = vuln.get("id") + if not vuln_id or vuln_id == '': # Skip empty ids + continue + vuln_cve = vuln.get("CVE", '') + if vuln_cve == '': + vuln_cve = vuln.get("cve", '') + if vuln_id.upper().startswith("CPE:"): + fd['cpe'] = vuln_id # Save the component CPE if we have one + if vuln_cve != '': + vuln_id = vuln_cve + vd = vdx.get(vuln_id) # Check if we've already encountered this vulnerability + if not vd: + vuln_source = vuln.get('source', '').lower() + vd = {'cve': vuln_cve, + 'source': 'NVD' if vuln_source == 'nvd' else 'GitHub Advisories', + 'url': f'https://nvd.nist.gov/vuln/detail/{vuln_cve}' if vuln_source == 'nvd' else f'https://github.com/advisories/{vuln_id}', + 'severity': self._sev_lookup(vuln.get('severity', 'unknown').lower()), + 'affects': set() + } + vd.get('affects').add(purl) + vdx[vuln_id] = vd if cdx.get(purl): self.print_debug(f'Component {purl} already stored: {cdx.get(purl)}') continue - fd = {} for field in ['id', 'vendor', 'component', 'version', 'latest']: fd[field] = d.get(field) licenses = d.get('licenses') @@ -113,7 +140,9 @@ def parse(self, data: json): fdl.append({'id': lic.get("name")}) fd['licenses'] = fdl cdx[purl] = fd - return cdx + # self.print_stderr(f'VD: {vdx}') + # self.print_stderr(f'CDX: {cdx}') + return cdx, vdx def produce_from_file(self, json_file: str, output_file: str = None) -> bool: """ @@ -139,7 +168,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: :param output_file: Output file (optional) :return: True if successful, False otherwise """ - cdx = self.parse(data) + cdx, vdx = self.parse(data) if not cdx: self.print_stderr('ERROR: No CycloneDX data returned for the JSON string provided.') return False @@ -154,7 +183,8 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'specVersion': '1.4', 'serialNumber': f'urn:uuid:{uuid.uuid4()}', 'version': 1, - 'components': [] + 'components': [], + 'vulnerabilities': [] } for purl in cdx: comp = cdx.get(purl) @@ -172,16 +202,38 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: lic_text.append({'license': {'name': lc_id}}) # Not an SPDX license, so store it by name else: lic_text.append({'license': {'id': spdx_id}}) - m_type = 'library' - data['components'].append({ - 'type': m_type, + c_data = { + 'type': 'library', 'name': comp.get('component'), 'publisher': comp.get('vendor', ''), 'version': comp.get('version'), 'purl': purl, + 'bom-ref': purl, 'licenses': lic_text - }) + } + cpe = comp.get('cpe', '') + if cpe and cpe != '': + c_data['cpe'] = cpe + data['components'].append(c_data) # End for loop + if vdx: + for vuln_id in vdx: + vulns = vdx.get(vuln_id) + if not vulns: + continue + v_source = vulns.get('source') + affects = [] + for purl in vulns.get('affects'): + affects.append({'ref': purl}) + vd = { + 'id': vuln_id, + 'source': {'name': v_source, 'url': vulns.get('url')}, + 'ratings': [{'severity': vulns.get('severity', 'unknown')}], + 'affects': affects + } + data['vulnerabilities'].append(vd) + # End for loop + file = sys.stdout if not output_file and self.output_file: output_file = self.output_file @@ -209,6 +261,24 @@ def produce_from_str(self, json_str: str, output_file: str = None) -> bool: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return False return self.produce_from_json(data, output_file) + + @staticmethod + def _sev_lookup(value: str): + """ + Lookup the given severity and return a CycloneDX valid version + :param value: severity to lookup + :return: CycloneDX severity + """ + return { + 'critical': 'critical', + 'high': 'high', + 'medium': 'medium', + 'moderate': 'medium', + 'low': 'low', + 'info': 'info', + 'none': 'none', + 'unknown': 'unknown' + }.get(value, 'unknown') # # End of CycloneDX Class # From 885f49be9f5a8e4313c221a3e673aba102f32d8e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 9 Nov 2022 12:03:35 +0000 Subject: [PATCH 025/489] Added obfuscation to winnowing/scanning --- src/scanoss/cli.py | 16 ++++++++--- src/scanoss/scanner.py | 32 ++++++++++++--------- src/scanoss/winnowing.py | 61 ++++++++++++++++++++++++---------------- 3 files changed, 67 insertions(+), 42 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 6bb6d1a9..80218bbc 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -90,6 +90,7 @@ def setup_args() -> None: p_scan.add_argument('--all-extensions', action='store_true', help='Scan all file extensions') p_scan.add_argument('--all-folders', action='store_true', help='Scan all folders') p_scan.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') + p_scan.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') p_scan.add_argument('--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' ) @@ -105,6 +106,8 @@ def setup_args() -> None: p_wfp.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) + p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') + p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') # Sub-command: dependency p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], @@ -172,7 +175,9 @@ def wfp(parser, args): if args.output: scan_output = args.output open(scan_output, 'w').close() - scanner = Scanner(debug=args.debug, quiet=args.quiet) + + scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value # Skip snippet generation or not + scanner = Scanner(debug=args.debug, quiet=args.quiet, obfuscate=args.obfuscate, scan_options=scan_options) if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') @@ -277,6 +282,8 @@ def scan(parser, args): print_stderr(f'Changing scanning POST size to: {args.post_size}k...') if args.timeout != 120: print_stderr(f'Changing scanning POST timeout to: {args.timeout}...') + if args.obfuscate: + print_stderr("Obfuscating file fingerprints...") elif not args.quiet: if args.timeout < 5: print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') @@ -292,7 +299,8 @@ def scan(parser, args): flags=flags, nb_threads=args.threads, post_size=args.post_size, timeout=args.timeout, no_wfp_file=args.no_wfp_output, all_extensions=args.all_extensions, all_folders=args.all_folders, hidden_files_folders=args.all_hidden, - scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, grpc_url=args.api2url + scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, + grpc_url=args.api2url, obfuscate=args.obfuscate ) if args.wfp: if not scanner.is_file_or_snippet_scan(): @@ -307,10 +315,10 @@ def scan(parser, args): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') exit(1) if os.path.isdir(args.scan_dir): - if not scanner.scan_folder_with_options(args.scan_dir): + if not scanner.scan_folder_with_options(args.scan_dir, scanner.winnowing.file_map): exit(1) elif os.path.isfile(args.scan_dir): - if not scanner.scan_file_with_options(args.scan_dir): + if not scanner.scan_file_with_options(args.scan_dir, scanner.winnowing.file_map): exit(1) else: print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index a81ab8aa..e82739d9 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -92,7 +92,8 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str sbom_path: str = None, scan_type: str = None, flags: str = None, nb_threads: int = 5, post_size: int = 64, timeout: int = 120, no_wfp_file: bool = False, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, - scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None + scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, + obfuscate: bool = False ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning @@ -111,7 +112,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str ver_details = self.__version_details() self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, - all_extensions=all_extensions + all_extensions=all_extensions, obfuscate=obfuscate ) self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, @@ -303,10 +304,11 @@ def is_dependency_scan(self): return True return False - def scan_folder_with_options(self, scan_dir: str) -> bool: + def scan_folder_with_options(self, scan_dir: str, file_map: dict = None) -> bool: """ Scan the given folder for whatever scaning options that have been configured :param scan_dir: directory to scan + :param file_map: mapping of obfuscated files back into originals :return: True if successful, False otherwise """ success = True @@ -326,7 +328,7 @@ def scan_folder_with_options(self, scan_dir: str) -> bool: if not self.scan_folder(scan_dir): success = False if self.threaded_scan: - if not self.__finish_scan_threaded(): + if not self.__finish_scan_threaded(file_map): success = False return success @@ -427,9 +429,10 @@ def __run_scan_threaded(self, scan_started: bool, file_count: int) -> bool: success = False return success - def __finish_scan_threaded(self) -> bool: + def __finish_scan_threaded(self, file_map: dict = None) -> bool: """ Wait for the threaded scans to complete + :param file_map: mapping of obfuscated files back into originals :return: True if successful, False otherwise """ success = True @@ -457,6 +460,10 @@ def __finish_scan_threaded(self) -> bool: for scan_resp in responses: if scan_resp is not None: for key, value in scan_resp.items(): + if file_map: # We have a map for obfuscated files. Check if we can revert it + fm = file_map.get(key) + if fm: + key = fm # Replace the obfuscated filename if first: raw_output += " \"%s\":%s" % (key, json.dumps(value, indent=2)) first = False @@ -513,10 +520,11 @@ def __finish_scan_threaded(self) -> bool: return success - def scan_file_with_options(self, file: str) -> bool: + def scan_file_with_options(self, file: str, file_map: dict = None) -> bool: """ Scan the given file for whatever scaning options that have been configured :param file: file to scan + :param file_map: mapping of obfuscated files back into originals :return: True if successful, False otherwise """ success = True @@ -536,7 +544,7 @@ def scan_file_with_options(self, file: str) -> bool: if not self.scan_file(file): success = False if self.threaded_scan: - if not self.__finish_scan_threaded(): + if not self.__finish_scan_threaded(file_map): success = False return success @@ -670,13 +678,11 @@ def scan_wfp_file(self, file: str = None) -> bool: return success - def scan_wfp_file_threaded(self, file: str = None) -> bool: + def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> bool: """ Scan the contents of the specified WFP file (threaded) - Parameters - ---------- - file: str - WFP file to scan (optional) + :param file: WFP file to scan (optional) + :param file_map: mapping of obfuscated files back into originals (optional) return: True if successful, False otherwise """ success = True @@ -723,7 +729,7 @@ def scan_wfp_file_threaded(self, file: str = None) -> bool: if not self.__run_scan_threaded(scan_started, file_count): success = False - elif not self.__finish_scan_threaded(): + elif not self.__finish_scan_threaded(file_map): success = False return success diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index cc0e58d7..6f260cb5 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -28,7 +28,7 @@ https://theory.stanford.edu/~aiken/publications/papers/sigmod03.pdf """ import hashlib -import sys +import pathlib from crc32c import crc32c from binaryornot.check import is_binary @@ -53,11 +53,11 @@ MIN_FILE_SIZE = 256 SKIP_SNIPPET_EXT = { # File extensions to ignore snippets for - ".exe", ".zip", ".tar", ".tgz", ".gz", ".7z", ".rar", ".jar", ".war", ".ear", ".class", ".pyc", - ".o", ".a", ".so", ".obj", ".dll", ".lib", ".out", ".app", ".bin", - ".lst", ".dat", ".json", ".htm", ".html", ".xml", ".md", ".txt", - ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".pages", ".key", ".numbers", - ".pdf", ".min.js", ".mf", ".sum" + ".exe", ".zip", ".tar", ".tgz", ".gz", ".7z", ".rar", ".jar", ".war", ".ear", ".class", ".pyc", + ".o", ".a", ".so", ".obj", ".dll", ".lib", ".out", ".app", ".bin", + ".lst", ".dat", ".json", ".htm", ".html", ".xml", ".md", ".txt", + ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".pages", ".key", ".numbers", + ".pdf", ".min.js", ".mf", ".sum" } @@ -105,7 +105,7 @@ class Winnowing(ScanossBase): """ def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = False, quiet: bool = False, - skip_snippets: bool = False, post_size: int = 64, all_extensions: bool = False + skip_snippets: bool = False, post_size: int = 64, all_extensions: bool = False, obfuscate: bool = False ): """ Instantiate Winnowing class @@ -115,10 +115,13 @@ def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = F Limit the size of a fingerprint to 64k (post size) - Default True """ super().__init__(debug, trace, quiet) - self.size_limit = size_limit + self.size_limit = size_limit self.skip_snippets = skip_snippets self.max_post_size = post_size * 1024 if post_size > 0 else MAX_POST_SIZE self.all_extensions = all_extensions + self.obfuscate = obfuscate + self.ob_count = 1 + self.file_map = {} if obfuscate else None @staticmethod def __normalize(byte): @@ -162,18 +165,19 @@ def __skip_snippets(self, file: str, src: str) -> bool: self.print_trace(f'Skipping snippets due to file ending: {file} - {ending}') return True; src_len = len(src) - if src_len == 0 or src_len <= MIN_FILE_SIZE: # Ignore empty or files that are too small + if src_len == 0 or src_len <= MIN_FILE_SIZE: # Ignore empty or files that are too small self.print_trace(f'Skipping snippets as the file is too small: {file} - {src_len}') return True - prefix = src[0:(MIN_FILE_SIZE-1)].lower().strip() - if len(prefix) > 0 and (prefix[0] == "{" or prefix[0] == "["): # Ignore json + prefix = src[0:(MIN_FILE_SIZE - 1)].lower().strip() + if len(prefix) > 0 and (prefix[0] == "{" or prefix[0] == "["): # Ignore json self.print_trace(f'Skipping snippets as the file appears to be JSON: {file}') return True - if prefix.startswith(" MAX_LONG_LINE_CHARS: # Ignore long lines + return True # Ignore xml & html & ac3d + index = src.index('\n') if '\n' in src else (src_len - 1) # TODO still necessary if we have a binary check? + if len(src[0:index]) > MAX_LONG_LINE_CHARS: # Ignore long lines self.print_trace(f'Skipping snippets due to file line being too long: {file} - {MAX_LONG_LINE_CHARS}') return True return False @@ -212,10 +216,9 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: Generate a Winnowing Finger Print (WFP) for the given file contents Parameters ---------- - file: str - file to fingerprint - contents: bytes - file contents + :param file: file to fingerprint + :param bin_file: binary file or not + :param contents: file contents Return ------ WFP string @@ -223,7 +226,13 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: file_md5 = hashlib.md5(contents).hexdigest() # Print file line content_length = len(contents) - wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, file) + wfp_filename = file + if self.obfuscate: # hide the real size of the file and its name, but keep the suffix + wfp_filename = f'{self.ob_count}{pathlib.Path(file).suffix}' + self.ob_count = self.ob_count + 1 + self.file_map[wfp_filename] = file # Save the file name map for later (reverse lookup) + + wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename) # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')): return wfp @@ -234,7 +243,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: last_hash = MAX_CRC32 last_line = 0 output = "" - # Otherwise recurse src_content and calculate Winnowing hashes + # Otherwise, recurse src_content and calculate Winnowing hashes for byte in contents: if byte == ASCII_LF: line += 1 @@ -262,10 +271,11 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: if last_line != line: if output: if self.size_limit and \ - (len(wfp.encode("utf-8")) + len(output.encode("utf-8"))) > self.max_post_size: + (len(wfp.encode("utf-8")) + len( + output.encode("utf-8"))) > self.max_post_size: self.print_debug(f'Truncating WFP (64k limit) for: {file}') output = '' - break # Stop collecting snippets as it's over 64k + break # Stop collecting snippets as it's over 64k wfp += output + '\n' output = "%d=%s" % (line, crc_hex) else: @@ -277,11 +287,12 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: window.pop(0) # Shift gram gram = gram[1:] - if output and (not self.size_limit or (len(wfp.encode("utf-8")) + len(output.encode("utf-8"))) < self.max_post_size): + if output and ( + not self.size_limit or (len(wfp.encode("utf-8")) + len(output.encode("utf-8"))) < self.max_post_size): wfp += output + '\n' return wfp # # End of Winnowing Class -# \ No newline at end of file +# From f94e07ae13282a29cb25d7d3855fa9c60e230b73 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 9 Nov 2022 12:04:08 +0000 Subject: [PATCH 026/489] version 1.2 release of obfuscation and CDX vulnerabilities --- CHANGELOG.md | 7 +++++++ Makefile | 2 ++ src/scanoss/__init__.py | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 287c391e..ca9b5dd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.2.0] - 2022-11-08 +### Added +- Added vulnerability reporting to CycloneDX output +- Added obfuscation to fingerprinting (--obfuscate) +- Added obfuscation to scanning (--obfuscate) + ## [1.1.1] - 2022-10-19 ### Fixed - Fixed issue with dependency parsing of yarn.lock files @@ -122,3 +128,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.0.6]: https://github.com/scanoss/scanoss.py/compare/v1.0.4...v1.0.6 [1.1.0]: https://github.com/scanoss/scanoss.py/compare/v1.0.6...v1.1.0 [1.1.1]: https://github.com/scanoss/scanoss.py/compare/v1.1.0...v1.1.1 +[1.2.0]: https://github.com/scanoss/scanoss.py/compare/v1.1.1...v1.2.0 diff --git a/Makefile b/Makefile index 06f0c387..613a4ec8 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,8 @@ publish: ## Publish Python package to PyPI @echo "Publishing package to PyPI..." twine upload dist/* +package_all: dist publish ## Build & Publish Python package to PyPI + ghcr_build: dist ## Build GitHub container image @echo "Building GHCR container image..." docker build --no-cache -t $(GHCR_FULLNAME) --platform linux/amd64 . diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 198a16b9..bbc17eba 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.1.1' +__version__ = '1.2.0' From 774e2d8e29a4157bb548394743e156f081e8d342 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 16 Nov 2022 19:11:16 +0000 Subject: [PATCH 027/489] fix spelling mistake --- src/scanoss/winnowing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 6f260cb5..d8d3c267 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -213,7 +213,7 @@ def is_binary(self, path: str): def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: """ - Generate a Winnowing Finger Print (WFP) for the given file contents + Generate a Winnowing fingerprint (WFP) for the given file contents Parameters ---------- :param file: file to fingerprint From 4e3be4135980599205c6b70b0fe07eacbadcbaab Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 16 Nov 2022 19:13:58 +0000 Subject: [PATCH 028/489] added file_count sub command --- CHANGELOG.md | 5 ++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 56 +++++++++++-- src/scanoss/filecount.py | 166 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 8 deletions(-) create mode 100644 src/scanoss/filecount.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ca9b5dd7..5b50c6eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.2.1] - 2022-11-11 +### Added +- Added sub-command (file_count)to produce a file summary (extensions & size) into a CSV + ## [1.2.0] - 2022-11-08 ### Added - Added vulnerability reporting to CycloneDX output @@ -129,3 +133,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.1.0]: https://github.com/scanoss/scanoss.py/compare/v1.0.6...v1.1.0 [1.1.1]: https://github.com/scanoss/scanoss.py/compare/v1.1.0...v1.1.1 [1.2.0]: https://github.com/scanoss/scanoss.py/compare/v1.1.1...v1.2.0 +[1.2.1]: https://github.com/scanoss/scanoss.py/compare/v1.2.0...v1.2.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index bbc17eba..b0c1b4b3 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.2.0' +__version__ = '1.2.1' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 80218bbc..46247e6b 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -30,6 +30,7 @@ from .winnowing import Winnowing from .scancodedeps import ScancodeDeps from .scantype import ScanType +from .filecount import FileCount from . import __version__ @@ -111,8 +112,8 @@ def setup_args() -> None: # Sub-command: dependency p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], - description=f'Produce dependency file summary: {__version__}', - help='Scan source code for dependencies, but do not decorate them') + description=f'Produce dependency file summary: {__version__}', + help='Scan source code for dependencies, but do not decorate them') p_dep.set_defaults(func=dependency) p_dep.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_dep.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) @@ -121,6 +122,15 @@ def setup_args() -> None: help='Timeout (in seconds) for scancode to complete (optional - default 600)' ) + # Sub-command: file_count + p_fc = subparsers.add_parser('file_count', aliases=['fc'], + description=f'Produce a file type count summary: {__version__}', + help='Search the source tree and produce a file type summary') + p_fc.set_defaults(func=file_count) + p_fc.add_argument('scan_dir', metavar='DIR', type=str, nargs='?', help='A folder to search') + p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) + p_fc.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') + # Global command options for p in [p_scan]: p.add_argument('--key', '-k', type=str, @@ -129,10 +139,12 @@ def setup_args() -> None: p.add_argument('--apiurl', type=str, help='SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)' ) - p.add_argument('--api2url', type=str, - help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' - ) - for p in [p_scan, p_wfp, p_dep]: + p.add_argument('--api2url', type=str, + help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' + ) + p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') + + for p in [p_scan, p_wfp, p_dep, p_fc]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -156,6 +168,36 @@ def ver(parser, args): """ print(f'Version: {__version__}') +def file_count(parser, args): + """ + Run the "file_count" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if not args.scan_dir: + print_stderr('Please specify a folder') + parser.parse_args([args.subparser, '-h']) + exit(1) + scan_output: str = None + if args.output: + scan_output = args.output + open(scan_output, 'w').close() + + counter = FileCount(debug=args.debug, quiet=args.quiet, trace=args.trace, scan_output=scan_output, + hidden_files_folders=args.all_hidden + ) + if not os.path.exists(args.scan_dir): + print_stderr(f'Error: Folder specified does not exist: {args.scan_dir}.') + exit(1) + if os.path.isdir(args.scan_dir): + counter.count_files(args.scan_dir) + else: + print_stderr(f'Error: Path specified is not a folder: {args.scan_dir}.') + exit(1) def wfp(parser, args): """ @@ -300,7 +342,7 @@ def scan(parser, args): timeout=args.timeout, no_wfp_file=args.no_wfp_output, all_extensions=args.all_extensions, all_folders=args.all_folders, hidden_files_folders=args.all_hidden, scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, - grpc_url=args.api2url, obfuscate=args.obfuscate + grpc_url=args.api2url, obfuscate=args.obfuscate, ignore_cert_errors=args.ignore_cert_errors ) if args.wfp: if not scanner.is_file_or_snippet_scan(): diff --git a/src/scanoss/filecount.py b/src/scanoss/filecount.py new file mode 100644 index 00000000..0d8db090 --- /dev/null +++ b/src/scanoss/filecount.py @@ -0,0 +1,166 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import csv +import os +import pathlib +import sys + +from progress.spinner import Spinner + +from .scanossbase import ScanossBase + + +class FileCount(ScanossBase): + """ + SCANOSS File Type Count class + Handle the scanning of files, snippets and dependencies + """ + def __init__(self, scan_output: str = None, hidden_files_folders: bool = False, + debug: bool = False, trace: bool = False, quiet: bool = False + ): + """ + Initialise scanning class + """ + super().__init__(debug, trace, quiet) + self.scan_output = scan_output + self.isatty = sys.stderr.isatty() + self.hidden_files_folders = hidden_files_folders + + def __filter_files(self, files: list) -> list: + """ + Filter which files should be considered for processing + :param files: list of files to filter + :return list of filtered files + """ + file_list = [] + for f in files: + ignore = False + if f.startswith(".") and not self.hidden_files_folders: # Ignore all . files unless requested + ignore = True + if not ignore: + file_list.append(f) + return file_list + + def __filter_dirs(self, dirs: list) -> list: + """ + Filter which folders should be considered for processing + :param dirs: list of directories to filter + :return: list of filtered directories + """ + dir_list = [] + for d in dirs: + ignore = False + if d.startswith(".") and not self.hidden_files_folders: # Ignore all . folders unless requested + ignore = True + if not ignore: + dir_list.append(d) + return dir_list + + def __log_result(self, string, outfile=None): + """ + Logs result to file or STDOUT + """ + if not outfile and self.scan_output: + outfile = self.scan_output + if outfile: + with open(outfile, "a") as rf: + rf.write(string + '\n') + else: + print(string) + + def count_files(self, scan_dir: str) -> bool: + """ + Search the specified folder producing counting the file types found. + + :param scan_dir str + Directory to scan + :return True if successful, False otherwise + """ + success = True + if not scan_dir: + raise Exception(f"ERROR: Please specify a folder to scan") + if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): + raise Exception(f"ERROR: Specified folder does not exist or is not a folder: {scan_dir}") + + self.print_msg(f'Searching {scan_dir} for files to count...') + spinner = None + if not self.quiet and self.isatty: + spinner = Spinner('Searching ') + file_types = {} + file_count = 0 + file_size = 0 + for root, dirs, files in os.walk(scan_dir): + self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') + dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories + filtered_files = self.__filter_files(files) # Strip out unwanted files + self.print_trace(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}') + for file in filtered_files: # Cycle through each filtered file + path = os.path.join(root, file) + f_size = 0 + try: + f_size = os.stat(path).st_size + except Exception as e: + self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # broken symlink + if f_size > 0: # Ignore broken links and empty files + file_count = file_count + 1 + file_size = file_size + f_size + f_suffix = pathlib.Path(file).suffix + if not f_suffix or f_suffix == '': + f_suffix = 'no_suffix' + self.print_trace(f'Counting {path} ({f_suffix} - {f_size})..') + fc = file_types.get(f_suffix) + if not fc: + fc = [1, f_size] + else: + fc[0] = fc[0] + 1 + fc[1] = fc[1] + f_size + file_types[f_suffix] = fc + if spinner: + spinner.next() + # End for loop + if spinner: + spinner.finish() + self.print_stderr(f'Found {file_count:,.0f} files with a total size of {file_size/(1<<20):,.2f} MB.') + if file_types: + csv_dict = [] + for k in file_types: + d = file_types[k] + csv_dict.append({'extension': k, 'count': d[0], 'size(MB)': f'{d[1]/(1<<20):,.2f}'}) + fields = ['extension', 'count', 'size(MB)'] + file = sys.stdout + if self.scan_output: + file = open(self.scan_output, 'w') + writer = csv.DictWriter(file, fieldnames=fields) + writer.writeheader() # writing headers (field names) + writer.writerows(csv_dict) # writing data rows + if self.scan_output: + file.close() + else: + FileCount.print_stderr(f'Warning: No files found to count in folder: {scan_dir}') + return success + + +# +# End of ScanOSS Class +# From 3cbbf53bb49d1f81a85d96ba997b4396e0eee767 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 18 Nov 2022 16:08:48 +0000 Subject: [PATCH 029/489] added REST SSL error ignore option --- setup.py | 5 ++++- src/scanoss/scanner.py | 6 +++--- src/scanoss/scanossapi.py | 22 ++++++++++++++++++---- src/scanoss/scanossgrpc.py | 4 ++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 7eee6281..3f384344 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,10 @@ def get_version(rel_path): description='Simple Python library to use the SCANOSS APIs.', long_description=read("PACKAGE.md"), long_description_content_type='text/markdown', - install_requires=["requests", "crc32c>=2.2", "binaryornot", "progress", "grpcio<=1.42.0", "protobuf>=3.16.0,<=3.19.1"], + install_requires=["requests", # TODO Add min req for python 3.10 here - urllib3>=1.26.8 and requests>=2.27.0? + "crc32c>=2.2", "binaryornot", "progress", "grpcio<=1.42.0", + "protobuf>=3.16.0,<=3.19.1" + ], include_package_data=True, package_data={'': ['data/*.json', 'data/*.txt']}, classifiers=[ diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index e82739d9..65669caa 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -85,7 +85,7 @@ class Scanner(ScanossBase): """ SCANOSS scanning class - Hanlde the scanning of files, snippets and dependencies + Handle the scanning of files, snippets and dependencies """ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str = 'plain', debug: bool = False, trace: bool = False, quiet: bool = False, api_key: str = None, url: str = None, @@ -93,7 +93,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str post_size: int = 64, timeout: int = 120, no_wfp_file: bool = False, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, - obfuscate: bool = False + obfuscate: bool = False, ignore_cert_errors: bool = False ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning @@ -116,7 +116,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str ) self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, - ver_details=ver_details + ver_details=ver_details, ignore_cert_errors=ignore_cert_errors ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 35f5156f..272d942e 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -31,6 +31,7 @@ import http.client as http_client from .scanossbase import ScanossBase +from requests.packages.urllib3.exceptions import InsecureRequestWarning DEFAULT_URL = "https://osskb.org/api/scan/direct" SCANOSS_SCAN_URL = os.environ.get("SCANOSS_SCAN_URL") if os.environ.get("SCANOSS_SCAN_URL") else DEFAULT_URL @@ -45,7 +46,7 @@ class ScanossApi(ScanossBase): def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None, url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - timeout: int = 120, ver_details: str = None): + timeout: int = 120, ver_details: str = None, ignore_cert_errors: bool = False): """ Initialise the SCANOSS API :param scan_type: Scan type (default identify) @@ -57,6 +58,10 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st :param debug: Enable debug (default False) :param trace: Enable trace (default False) :param quiet: Enable quite mode (default False) + + To set a custom certificate use: + REQUESTS_CA_BUNDLE=/path/to/cert.pem + SSL_CERT_FILE=/path/to/cert.pem """ super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_SCAN_URL @@ -66,6 +71,7 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st self.sbom_path = sbom_path self.flags = flags self.timeout = timeout if timeout > 5 else 120 + self.ignore_cert_errors = ignore_cert_errors self.headers = {} if ver_details: self.headers['x-scanoss-client'] = ver_details @@ -77,6 +83,9 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 + if self.ignore_cert_errors: + self.print_debug(f'Ignoring cert errors...') + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) def load_sbom(self): """ @@ -115,16 +124,21 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): try: r = None r = requests.post(self.url, files=scan_files, data=form_data, headers=self.headers, - timeout=self.timeout) + timeout=self.timeout, + verify=False if self.ignore_cert_errors else None + ) + except requests.exceptions.SSLError as e: + self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') + raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: if retry > 5: # Timed out 5 or more times, fail - self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data: {scan_files}') + self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data - {e}: {scan_files}') raise Exception(f"ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for {self.url}") from e else: self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...') time.sleep(5) except Exception as e: - self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data: {scan_files}') + self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}: {scan_files}') raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e else: if not r: diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 7c1e4f96..37c294b3 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -56,6 +56,10 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu :param trace: :param quiet: :param cert: + + To set a custom certificate use: + GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/certs/cert.pem + More details here: https://grpc.github.io/grpc/cpp/grpc__security__constants_8h.html """ super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_GRPC_URL From 20ccf9dbd2faad8ac2c9bb778a5e4d933ffa8296 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 18 Nov 2022 16:14:28 +0000 Subject: [PATCH 030/489] added multi-platform docker build --- CHANGELOG.md | 7 +++++++ Dockerfile | 16 +++++++++++----- Makefile | 36 ++++++++++++++++++++++++++++++------ src/scanoss/__init__.py | 2 +- 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b50c6eb..18db579d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.2.2] - 2022-11-18 +### Added +- Added SSL cert error ignore option (--ignore-cert-errors) for REST calls + Custom certificates can be supplied using environment variables +- Added multi-platform Docker images (AMD64 & ARM64) + ## [1.2.1] - 2022-11-11 ### Added - Added sub-command (file_count)to produce a file summary (extensions & size) into a CSV @@ -134,3 +140,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.1.1]: https://github.com/scanoss/scanoss.py/compare/v1.1.0...v1.1.1 [1.2.0]: https://github.com/scanoss/scanoss.py/compare/v1.1.1...v1.2.0 [1.2.1]: https://github.com/scanoss/scanoss.py/compare/v1.2.0...v1.2.1 +[1.2.2]: https://github.com/scanoss/scanoss.py/compare/v1.2.1...v1.2.2 diff --git a/Dockerfile b/Dockerfile index 51463163..27305c47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-buster as base +FROM python:3.10-slim-buster as base LABEL maintainer="SCANOSS " @@ -16,14 +16,20 @@ ENV PATH=/root/.local/bin:$PATH COPY ./dist/scanoss-*-py3-none-any.whl /install/ -#RUN pip3 install --user scanoss RUN pip3 install --user /install/scanoss-*-py3-none-any.whl RUN pip3 install --user scancode-toolkit-mini -RUN pip3 install --user typecode-libmagic +#RUN pip3 install --user typecode-libmagic + +# Download compile and install typecode-libmagic from source (as there is not ARM wheel available) +ADD https://github.com/nexB/typecode_libmagic_from_sources/archive/refs/tags/v5.39.210212.tar.gz /install/ +RUN tar -xvzf /install/v5.39.210212.tar.gz -C /install \ + && cd /install/typecode_libmagic_from_sources* \ + && ./build.sh && python3 setup.py sdist bdist_wheel \ + && pip3 install --user `ls /install/typecode_libmagic_from_sources*/dist/*.whl` # Remove license data references as they are not required for dependency scanning (to save space) -RUN rm -rf /root/.local/lib/python3.8/site-packages/licensedcode/data/rules /root/.local/lib/python3.8/site-packages/licensedcode/data/cache -RUN mkdir /root/.local/lib/python3.8/site-packages/licensedcode/data/rules /root/.local/lib/python3.8/site-packages/licensedcode/data/cache +RUN rm -rf /root/.local/lib/python3.10/site-packages/licensedcode/data/rules /root/.local/lib/python3.10/site-packages/licensedcode/data/cache +RUN mkdir /root/.local/lib/python3.10/site-packages/licensedcode/data/rules /root/.local/lib/python3.10/site-packages/licensedcode/data/cache FROM base diff --git a/Makefile b/Makefile index 613a4ec8..60c7a115 100644 --- a/Makefile +++ b/Makefile @@ -51,11 +51,19 @@ publish: ## Publish Python package to PyPI @echo "Publishing package to PyPI..." twine upload dist/* -package_all: dist publish ## Build & Publish Python package to PyPI +package_all: dist publish ## Build & Publish Python package to PyPI -ghcr_build: dist ## Build GitHub container image +ghcr_build: dist ## Build GitHub container image with local arch @echo "Building GHCR container image..." - docker build --no-cache -t $(GHCR_FULLNAME) --platform linux/amd64 . + docker build -t $(GHCR_FULLNAME) . + +ghcr_amd64: dist ## Build GitHub AMD64 container image + @echo "Building GHCR AMD64 container image..." + docker build -t $(GHCR_FULLNAME) --platform linux/amd64 . + +ghcr_arm64: dist ## Build GitHub ARM64 container image + @echo "Building GHCR ARM64 container image..." + docker build -t $(GHCR_FULLNAME) --platform linux/arm64 . ghcr_tag: ## Tag the latest GH container image with the version from Python @echo "Tagging GHCR latest image with $(VERSION)..." @@ -66,12 +74,24 @@ ghcr_push: ## Push the GH container image to GH Packages docker push $(GHCR_FULLNAME):$(VERSION) docker push $(GHCR_FULLNAME):latest -ghcr_all: ghcr_build ghcr_tag ghcr_push ## Execute all GitHub Package container actions +ghcr_release: dist ## Build/Publish GitHub multi-platform container image + @echo "Building & Releasing GHCR multi-platform container image $(VERSION)..." + docker buildx build --push -t $(GHCR_FULLNAME):$(VERSION) --platform linux/arm64,linux/amd64 . + +ghcr_all: ghcr_release ## Execute all GHCR container actions -docker_build: ## Build Docker container image +docker_build: ## Build Docker container image with local arch @echo "Building Docker image..." docker build --no-cache -t $(DOCKER_FULLNAME) . +docker_amd64: dist ## Build Docker AMD64 container image + @echo "Building Docker AMD64 container image..." + docker build -t $(DOCKER_FULLNAME) --platform linux/amd64 . + +docker_arm64: dist ## Build Docker ARM64 container image + @echo "Building Docker ARM64 container image..." + docker build -t $(DOCKER_FULLNAME) --platform linux/arm64 . + docker_tag: ## Tag the latest Docker container image with the version from Python @echo "Tagging Docker latest image with $(VERSION)..." docker tag $(DOCKER_FULLNAME):latest $(DOCKER_FULLNAME):$(VERSION) @@ -81,4 +101,8 @@ docker_push: ## Push the Docker container image to DockerHub docker push $(DOCKER_FULLNAME):$(VERSION) docker push $(DOCKER_FULLNAME):latest -docker_all: docker_build docker_tag docker_push ## Execute all DockerHub container actions +docker_release: dist ## Build/Publish Docker multi-platform container image + @echo "Building & Releasing Docker multi-platform container image $(VERSION)..." + docker buildx build --push -t $(DOCKER_FULLNAME):$(VERSION) --platform linux/arm64,linux/amd64 . + +docker_all: docker_release ## Execute all DockerHub container actions diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b0c1b4b3..38dd7d4e 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.2.1' +__version__ = '1.2.2' From 8f9e632ff299cb7efb2507b8d444574ef390a444 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 24 Nov 2022 08:08:38 +0000 Subject: [PATCH 031/489] Added max threaded scanning env override --- CHANGELOG.md | 7 +++++++ src/scanoss/__init__.py | 2 +- src/scanoss/threadedscanning.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18db579d..8bbb7144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.2.3] - 2022-11-22 +### Added +- Added Max Threaded scanning override env var (SCANOSS_MAX_ALLOWED_THREADS) + If the backend system can handle more than the current maximum (30), then set this env to that number + `export SCANOSS_MAX_ALLOWED_THREADS=40` + ## [1.2.2] - 2022-11-18 ### Added - Added SSL cert error ignore option (--ignore-cert-errors) for REST calls @@ -141,3 +147,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.2.0]: https://github.com/scanoss/scanoss.py/compare/v1.1.1...v1.2.0 [1.2.1]: https://github.com/scanoss/scanoss.py/compare/v1.2.0...v1.2.1 [1.2.2]: https://github.com/scanoss/scanoss.py/compare/v1.2.1...v1.2.2 +[1.2.3]: https://github.com/scanoss/scanoss.py/compare/v1.2.2...v1.2.3 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 38dd7d4e..037fd279 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.2.2' +__version__ = '1.2.3' diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index 20e21cba..9d8223a9 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -35,7 +35,7 @@ from .scanossbase import ScanossBase WFP_FILE_START = "file=" -MAX_ALLOWED_THREADS = 30 +MAX_ALLOWED_THREADS = int(os.environ.get("SCANOSS_MAX_ALLOWED_THREADS")) if os.environ.get("SCANOSS_MAX_ALLOWED_THREADS") else 30 @dataclass class ThreadedScanning(ScanossBase): From 6713e0623380375cd550c144d11980fa769aae40 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 2 Dec 2022 12:45:12 +0000 Subject: [PATCH 032/489] Added custom certificate loading --- src/scanoss/scanossgrpc.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 37c294b3..34d9acd5 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -36,7 +36,6 @@ from .scanossbase import ScanossBase # DEFAULT_URL = "https://osskb.org" -# DEFAULT_URL = "localhost:50051" DEFAULT_URL = "https://scanoss.com" SCANOSS_GRPC_URL = os.environ.get("SCANOSS_GRPC_URL") if os.environ.get("SCANOSS_GRPC_URL") else DEFAULT_URL SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' @@ -48,7 +47,7 @@ class ScanossGrpc(ScanossBase): """ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - cert: bytes = None, api_key: str = None, ver_details: str = None): + ca_cert: str = None, api_key: str = None, ver_details: str = None): """ :param url: @@ -60,6 +59,9 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu To set a custom certificate use: GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/certs/cert.pem More details here: https://grpc.github.io/grpc/cpp/grpc__security__constants_8h.html + https://github.com/grpc/grpc/blob/master/doc/environment_variables.md + To enable a Proxy use: + grpc_proxy='http://:' """ super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_GRPC_URL @@ -77,14 +79,16 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu if port is None: port = 443 if u.scheme == 'https' else 80 # Set the default port number if it's not available self.url = f'{u.hostname}:{port}' - if cert is not None: + cert_data = None + if ca_cert is not None: secure = True + cert_data = ScanossGrpc._load_cert(ca_cert) self.print_debug(f'Setting up (secure: {secure}) connection to {self.url}...') if secure is False: self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) # insecure connection else: - if cert is not None: - credentials = grpc.ssl_channel_credentials(cert) # secure with specified certificate + if ca_cert is not None: + credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate else: credentials = grpc.ssl_channel_credentials() # secure connection with default certificate self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) @@ -179,6 +183,13 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st self.print_stderr(f'Not such a success (rqId: {request_id}): {status_response.message}') return False return True + + @staticmethod + def _load_cert(cert_file: str) -> bytes: + certificate_chain = None + with open(cert_file, 'rb') as f: + certificate_chain = f.read() + return certificate_chain # # End of ScanossGrpc Class # From c722594385767ec25cc198b5e879bd41ed475bb0 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 2 Dec 2022 12:45:32 +0000 Subject: [PATCH 033/489] added explicit urllib3 dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index db3b752f..99577408 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ binaryornot progress grpcio<=1.42.0 protobuf>=3.16.0,<=3.19.1 +urllib3 \ No newline at end of file From 2ee7f2421253c968f6faf62bc1c5f9736710f575 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 2 Dec 2022 13:04:39 +0000 Subject: [PATCH 034/489] added tmp folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 27297b35..2de07196 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ github.com_* master.zip dist/ build/ +tmp/ .eggs *.egg-info __pycache__ From ec4f33ecd9fe98c1a0b17b5f31eecb3134ea30cc Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 2 Dec 2022 15:57:04 +0000 Subject: [PATCH 035/489] added snippet line details --- src/scanoss/csvoutput.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index be1d23ad..6974039e 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -62,6 +62,9 @@ def parse(self, data: json): id_details = d.get("id") if not id_details or id_details == 'none': continue + matched = d.get("matched", '') + lines = d.get("lines", '') + oss_lines = d.get("oss_lines", '') detected = {} if id_details == 'dependency': dependencies = d.get("dependencies") @@ -118,7 +121,8 @@ def parse(self, data: json): 'detected_component': detected.get('component'), 'detected_license': detected.get('licenses'), 'detected_version': detected.get('version'), 'detected_latest': detected.get('latest'), - 'detected_purls': detected.get('purls') + 'detected_purls': detected.get('purls'), + 'detected_match': matched, 'detected_lines': lines, 'detected_oss_lines': oss_lines }) row_id = row_id + 1 return csv_dict @@ -153,7 +157,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: return False # Header row/column details fields = ['inventory_id', 'path', 'detected_usage', 'detected_component', 'detected_license', 'detected_version', - 'detected_latest', 'detected_purls'] + 'detected_latest', 'detected_purls', 'detected_match', 'detected_lines', 'detected_oss_lines'] file = sys.stdout if not output_file and self.output_file: output_file = self.output_file From f8432a3762ade34c989b1b82f50cb1f59c43e7b4 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 2 Dec 2022 15:58:09 +0000 Subject: [PATCH 036/489] added proxy and cert options --- src/scanoss/scanner.py | 7 ++++--- src/scanoss/scanossapi.py | 27 +++++++++++++++++++-------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 65669caa..5d261265 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -93,7 +93,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str post_size: int = 64, timeout: int = 120, no_wfp_file: bool = False, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, - obfuscate: bool = False, ignore_cert_errors: bool = False + obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, ca_cert: str = None ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning @@ -116,11 +116,12 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str ) self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, - ver_details=ver_details, ignore_cert_errors=ignore_cert_errors + ver_details=ver_details, ignore_cert_errors=ignore_cert_errors, + proxy=proxy, ca_cert=ca_cert ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, - ver_details=ver_details + ver_details=ver_details, ca_cert=ca_cert ) self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace) self.nb_threads = nb_threads diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 272d942e..4de1340e 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -29,11 +29,13 @@ import requests import uuid import http.client as http_client +import urllib3 +from urllib3.exceptions import InsecureRequestWarning from .scanossbase import ScanossBase -from requests.packages.urllib3.exceptions import InsecureRequestWarning -DEFAULT_URL = "https://osskb.org/api/scan/direct" +DEFAULT_URL = "https://osskb.org/api/scan/direct" # default free service URL +DEFAULT_URL2 = "https://scanoss.com/api/scan/direct" # default premium service URL SCANOSS_SCAN_URL = os.environ.get("SCANOSS_SCAN_URL") if os.environ.get("SCANOSS_SCAN_URL") else DEFAULT_URL SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' @@ -46,7 +48,8 @@ class ScanossApi(ScanossBase): def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None, url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - timeout: int = 120, ver_details: str = None, ignore_cert_errors: bool = False): + timeout: int = 120, ver_details: str = None, ignore_cert_errors: bool = False, + proxy: str = None, ca_cert: str = None): """ Initialise the SCANOSS API :param scan_type: Scan type (default identify) @@ -61,11 +64,15 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st To set a custom certificate use: REQUESTS_CA_BUNDLE=/path/to/cert.pem - SSL_CERT_FILE=/path/to/cert.pem + To enable a Proxy use: + HTTP_PROXY='http://:' + HTTPS_PROXY='http://:' """ super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_SCAN_URL self.api_key = api_key if api_key else SCANOSS_API_KEY + if self.api_key and not url and not os.environ.get("SCANOSS_SCAN_URL"): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.scan_type = scan_type self.scan_format = scan_format if scan_format else 'plain' self.sbom_path = sbom_path @@ -83,9 +90,14 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 + self.verify = None if self.ignore_cert_errors: self.print_debug(f'Ignoring cert errors...') - requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + urllib3.disable_warnings(InsecureRequestWarning) + self.verify = False + elif ca_cert: + self.verify = ca_cert + self.proxies = {'https': proxy, 'http': proxy} if proxy else None def load_sbom(self): """ @@ -124,10 +136,9 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): try: r = None r = requests.post(self.url, files=scan_files, data=form_data, headers=self.headers, - timeout=self.timeout, - verify=False if self.ignore_cert_errors else None + timeout=self.timeout, verify=self.verify, proxies=self.proxies ) - except requests.exceptions.SSLError as e: + except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e: self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: From 404f7e70684e8b61952172f2a0dcbfeb85c98458 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 2 Dec 2022 15:59:01 +0000 Subject: [PATCH 037/489] added convert, utils commands and proxy/cert support --- CHANGELOG.md | 11 ++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 112 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 120 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bbb7144..ec82ea90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.0] - 2022-12-02 +### Added +- Added support for proxy (--proxy) and certificates (--ca-certs) while scanning + - Certificates can also be supplied using environment variables: REQUESTS_CA_BUNDLE & GRPC_DEFAULT_SSL_ROOTS_FILE_PATH + - Proxies can be supplied using: grpc_proxy, https_proxy, http_proxy, HTTPS_PROXY, HTTP_PROXY +- Added snippet match fields to CSV output +- Added `convert` command to convert raw JSON reports into CSV, CycloneDX and SPDXLite +- Added `utils certloc` sub-command to print the location of Python's CA Cert file + - This is useful to know where to append custom certificates to if needed + ## [1.2.3] - 2022-11-22 ### Added - Added Max Threaded scanning override env var (SCANOSS_MAX_ALLOWED_THREADS) @@ -148,3 +158,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.2.1]: https://github.com/scanoss/scanoss.py/compare/v1.2.0...v1.2.1 [1.2.2]: https://github.com/scanoss/scanoss.py/compare/v1.2.1...v1.2.2 [1.2.3]: https://github.com/scanoss/scanoss.py/compare/v1.2.2...v1.2.3 +[1.3.0]: https://github.com/scanoss/scanoss.py/compare/v1.2.3...v1.3.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 037fd279..343d94b4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.2.3' +__version__ = '1.3.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 46247e6b..06df545e 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -31,6 +31,9 @@ from .scancodedeps import ScancodeDeps from .scantype import ScanType from .filecount import FileCount +from .cyclonedx import CycloneDx +from .spdxlite import SpdxLite +from .csvoutput import CsvOutput from . import __version__ @@ -46,6 +49,8 @@ def setup_args() -> None: Setup all the command line arguments for processing """ parser = argparse.ArgumentParser(description=f'SCANOSS Python CLI. Ver: {__version__}, License: MIT') + parser.add_argument('--version', '-v', action='store_true', help='Display version details') + subparsers = parser.add_subparsers(title='Sub Commands', dest='subparser', description='valid subcommands', help='sub-command help' ) @@ -131,6 +136,35 @@ def setup_args() -> None: p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) p_fc.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') + # Sub-command: convert + p_cnv = subparsers.add_parser('convert', aliases=['cv', 'cnv', 'cvrt'], + description=f'Convert results files between formats: {__version__}', + help='Convert file format') + p_cnv.set_defaults(func=convert) + p_cnv.add_argument('--input', '-i', type=str, required=True, help='Input file name') + p_cnv.add_argument('--output','-o', type=str, help='Output result file name (optional - default stdout).' ) + p_cnv.add_argument('--format','-f', type=str, choices=['cyclonedx', 'spdxlite', 'csv'], default='spdxlite', + help='Output format (optional - default: spdxlite)' + ) + p_cnv.add_argument('--input-format', type=str, choices=['plain'], default='plain', + help='Input format (optional - default: plain)' + ) + + # Sub-command: utils + p_util = subparsers.add_parser('utils', aliases=['ut', 'util'], + description=f'SCANOSS Utility commands: {__version__}', + help='General utility support commands') + + utils_sub = p_util.add_subparsers(title='Utils Commands', dest='utilsubparser', description='utils sub-commands', + help='utils sub-commands' + ) + + # Utils Sub-command: utils certloc + p_c_loc = utils_sub.add_parser('certloc', aliases=['cl'], + description=f'Show location of Python CA Certs: {__version__}', + help='Display the location of Python CA Certs') + p_c_loc.set_defaults(func=utils_certloc) + # Global command options for p in [p_scan]: p.add_argument('--key', '-k', type=str, @@ -142,16 +176,31 @@ def setup_args() -> None: p.add_argument('--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' ) + p.add_argument('--proxy', type=str, help='Proxy URL to use for connections (optional). ' + 'Can also use the environment variable "HTTPS_PROXY=:" ' + 'and "grcp_proxy=:" for gRPC' + ) + p.add_argument('--ca-cert', type=str, help='Alternative certificate PEM file (optional). ' + 'Can also use the environment variable ' + '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem" and ' + '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC' + ) p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') - for p in [p_scan, p_wfp, p_dep, p_fc]: + for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') args = parser.parse_args() + if args.version: + ver(parser, args) + exit(0) if not args.subparser: - parser.print_help() + parser.print_help() # No sub command subcommand, print general help + exit(1) + elif args.subparser == 'utils' and not args.utilsubparser: # No utils sub command supplied + parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed exit(1) args.func(parser, args) # Execute the function associated with the sub-command @@ -326,6 +375,10 @@ def scan(parser, args): print_stderr(f'Changing scanning POST timeout to: {args.timeout}...') if args.obfuscate: print_stderr("Obfuscating file fingerprints...") + if args.proxy: + print_stderr(f'Using Proxy {arg.proxy}...') + if args.ca_cert: + print_stderr(f'Using Certificate {arg.ca_cert}...') elif not args.quiet: if args.timeout < 5: print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') @@ -333,7 +386,9 @@ def scan(parser, args): if not os.access( os.getcwd(), os.W_OK ): # Make sure the current directory is writable. If not disable saving WFP print_stderr(f'Warning: Current directory is not writable: {os.getcwd()}') args.no_wfp_output = True - + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + exit(1) scan_options = get_scan_options(args) # Figure out what scanning options we have scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, api_key=args.key, url=args.apiurl, @@ -342,7 +397,8 @@ def scan(parser, args): timeout=args.timeout, no_wfp_file=args.no_wfp_output, all_extensions=args.all_extensions, all_folders=args.all_folders, hidden_files_folders=args.all_hidden, scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, - grpc_url=args.api2url, obfuscate=args.obfuscate, ignore_cert_errors=args.ignore_cert_errors + grpc_url=args.api2url, obfuscate=args.obfuscate, + ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, ca_cert=args.ca_cert ) if args.wfp: if not scanner.is_file_or_snippet_scan(): @@ -397,6 +453,54 @@ def dependency(parser, args): if not sc_deps.get_dependencies(what_to_scan=args.scan_dir, result_output=scan_output): exit(1) +def convert(parser, args): + """ + Run the "convert" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if not args.input: + print_stderr('Please specify an input file to convert') + parser.parse_args([args.subparser, '-h']) + exit(1) + success = False + if args.format == 'cyclonedx': + if not args.quiet: + print_stderr(f'Producing CycloneDX report...') + cdx = CycloneDx(debug=args.debug, output_file=args.output) + success = cdx.produce_from_file(args.input) + elif args.format == 'spdxlite': + if not args.quiet: + print_stderr(f'Producing SPDX Lite report...') + spdxlite = SpdxLite(debug=args.debug, output_file=args.output) + success = spdxlite.produce_from_file(args.input) + elif args.format == 'csv': + if not args.quiet: + print_stderr(f'Producing CSV report...') + csvo = CsvOutput(debug=args.debug, output_file=args.output) + success = csvo.produce_from_file(args.input) + else: + print_stderr(f'ERROR: Unknown output format (--format): {args.format}') + if not success: + exit(1) + +def utils_certloc(parser, args): + """ + Run the "utils certloc" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + import certifi + print(f'CA Cert File: {certifi.where()}') + def main(): """ Run the ScanOSS CLI From 3149cd8bb6b90e891120f1bfebb70e5ffb9200df Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 6 Dec 2022 11:47:01 +0000 Subject: [PATCH 038/489] added client help file --- CLIENT_HELP.md | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 CLIENT_HELP.md diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md new file mode 100644 index 00000000..b684ee46 --- /dev/null +++ b/CLIENT_HELP.md @@ -0,0 +1,78 @@ +# SCANOSS Client Usage Help +This file contains useful tips/tricks for getting the most out of the SCANOSS platform using the Python client/SDK. + +## Certificate Management +The SCANOSS SaaS platform runs over HTTPS with publicly signed SSL certificates. +However, on-premise installations, or those with a proxy in the middle might be leveraging self-signed versions. + +This can cause issues for the SCANOSS clients. + +### Certificate Download +In order to connect to a self-signed endpoint, it's necessary to download that cert and add it to the trust store for the client. +The following is an OpenSSL-based command script which can produce this file: +```shell +cert_download.sh -n +``` +Simply pass in the hostname `-n scanoss.com` and optionally the port `-p 8443` (defaults to `443`) and it will produce a PEM file called `scanoss.com.pem`. + +The `scanoss-py` CLI also supports certificate download using this command: +```shell +scanoss-py utils cdl -n scanoss.com -o scanoss-com.pem +``` + +It is also possible to download the certificate using a web browser, for example FireFox. Simply browse to the site, view the certificate and choose to download. + +### Use Custom Certificate with CLI +There are a number of ways to leverage this custom certificate from the `scanoss-py` CLI. +- Environment Variables +- Command Line Options +- Appending to the default certificates + +#### Custom Certificate with Env Vars +The `scanoss-py` CLI uses two communication methods; REST & gRPC and as such requires two env vars to be set if following this method. +- REST - Use `REQUESTS_CA_BUNDLE` + - `export REQUESTS_CA_BUNDLE=/path/to/cert.pem` +- gRPC - Use `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` + - `export GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cert.pem` + +#### Custom Certificate with CLI Options +The `scanoss-py` CLI has a `--ca-cert` option to allow the specification of a custom certificate file to be used when communicating over REST/gRPC. +Simply set it using: +```shell +scanoss-py scan --ca-cert scanoss-com.pem -o results.json . +``` +Alternative API Urls can also be configured (if necessary) using `--apiurl` & `api2url`. + +#### Custom Certificate appended to Defaults +It is also possible to append this custom certificate to the default certificate list used by `scanoss-py`. +This file location can be determined by using: +```shell +scanoss-py utils cl +``` +The resulting certificate file name can then be opened and the custom certificate appended to the end. +For example: +```shell +cat scanoss-com.pem >> /usr/local/lib/python3.10/site-packages/certifi/cacert.pem +``` + +## Proxy Configuration +The SCANOSS clients can be configured to work with proxies. There are a number of ways to achieve this: + +- Environment Variables +- Command Line Options + +### Proxy Env Vars +There are a number of environment variables that can be specified to force the `scanoss-y` command to route calls via proxy. + +- REST - `https_proxy`, `http_proxy`, `HTTPS_PROXY`, `HTTP_PROXY` +- gRPC - `grpc_proxy`, `https_proxy`, `http_proxy` + +Set the variable as follows: `export https_proxy="http://:` + +The REST client support both lowercase & uppercase proxy names, however the gRPC client only supports lowercase variants. The gRPC client provides one extra variable, `grpc_proxy` to enable a separate proxy to be leveraged for it alone. + +### Proxy CLI Options +The proxy for REST based calls can also be configured directly on the `scanoss-py` commandline using `--proxy`. For example: +```shell +scanoss-py scan --proxy "http::" -o results.json . +``` \ No newline at end of file From 361bd69ca63148047686bcb2ff814211cefe8ef4 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 6 Dec 2022 11:47:32 +0000 Subject: [PATCH 039/489] added certificate download script --- cert_download.sh | 103 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 cert_download.sh diff --git a/cert_download.sh b/cert_download.sh new file mode 100755 index 00000000..f2fd528e --- /dev/null +++ b/cert_download.sh @@ -0,0 +1,103 @@ +#!/bin/bash +### +# SPDX-License-Identifier: MIT +# +# Copyright (c) 2022, SCANOSS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +### +# +# Attempt to download an SSL certificate from the specified host and convert to a PEM file +# + +script_name=$(basename $0) + +help() +{ + echo "Usage: $script_name -n [-p ] [-o pem-file] [-f] [-h] + -n -- Hostname to download certificate from + -p -- Port number to use (default 443) + -o -- Output filename (default .pem) + -f -- Force the overwrite of existing pem file" + exit 2 +} + +SHORT=n:,p:o:,h,f +OPTS=$(getopt $SHORT "$@") +if [[ $? -ne 0 ]]; then + help +fi +VALID_ARGUMENTS=$# +if [ "$VALID_ARGUMENTS" -eq 0 ]; then # No arguments supplied, print help + help +fi +set -- $OPTS + +force=0 +while :; do +# echo "1: $1 - 2: $2" + case "$1" in + -n ) + host="$2" + shift 2 + ;; + -p ) + port="$2" + shift 2 + ;; + -o ) + pemfile="$2" + shift 2 + ;; + -f ) + force=1 + shift + ;; + -h ) + help + ;; + --) + shift; + break + ;; + *) + echo "Unexpected option: $1" + help + ;; + esac +done + +if [ -z "$host" ] ; then + echo "Error: Please provide a hostname -h " + exit 1 +fi +if [ -z "$port" ] ; then + port="443" +fi +if [ -z "$pemfile" ] ; then + pemfile="${host}.pem" +fi + +if [ $force -eq 0 ] && [ -f "$pemfile" ] ; then + echo "Error: Output PEM file already exists: $pemfile" + exit 1 +fi +echo "Attempting to get PEM certificate from $host:$port and saving to $pemfile ..." + +openssl s_client -showcerts -connect "$host:$port" -servername "$host" /dev/null | openssl x509 -outform PEM > "$pemfile" From ded7def23c3aff0ef50ab47a7541b0cf58ef799e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 6 Dec 2022 11:53:21 +0000 Subject: [PATCH 040/489] ignoring pem files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2de07196..e8aa8281 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.wfp *-result.json +*.pem .vscode/ gitee_com_* github.com_* From c15e10b6ffe8976a634d2d74a0a6b622d54a2360 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 7 Dec 2022 09:11:17 +0000 Subject: [PATCH 041/489] added certificate download and help documentation --- CHANGELOG.md | 7 +++++ CLIENT_HELP.md | 6 ++-- PACKAGE.md | 3 ++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 70 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 82 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec82ea90..797fde06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.1] - 2022-12-07 +### Added +- Added `utils cert-download` sub-command to help with the use of custom certificates + - Included a local certificate download script leveraging openssl too: [cert_download.sh](cert_download.sh) +- Added [documentation](CLIENT_HELP.md) to help with certificate and proxy configuration + ## [1.3.0] - 2022-12-02 ### Added - Added support for proxy (--proxy) and certificates (--ca-certs) while scanning @@ -159,3 +165,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.2.2]: https://github.com/scanoss/scanoss.py/compare/v1.2.1...v1.2.2 [1.2.3]: https://github.com/scanoss/scanoss.py/compare/v1.2.2...v1.2.3 [1.3.0]: https://github.com/scanoss/scanoss.py/compare/v1.2.3...v1.3.0 +[1.3.1]: https://github.com/scanoss/scanoss.py/compare/v1.3.0...v1.3.1 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index b684ee46..bdf820e5 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -62,17 +62,17 @@ The SCANOSS clients can be configured to work with proxies. There are a number o - Command Line Options ### Proxy Env Vars -There are a number of environment variables that can be specified to force the `scanoss-y` command to route calls via proxy. +There are a number of environment variables that can be specified to force the `scanoss-py` command to route calls via proxy. - REST - `https_proxy`, `http_proxy`, `HTTPS_PROXY`, `HTTP_PROXY` - gRPC - `grpc_proxy`, `https_proxy`, `http_proxy` -Set the variable as follows: `export https_proxy="http://:` +Set the variable as follows: `export https_proxy="http://:"` The REST client support both lowercase & uppercase proxy names, however the gRPC client only supports lowercase variants. The gRPC client provides one extra variable, `grpc_proxy` to enable a separate proxy to be leveraged for it alone. ### Proxy CLI Options The proxy for REST based calls can also be configured directly on the `scanoss-py` commandline using `--proxy`. For example: ```shell -scanoss-py scan --proxy "http::" -o results.json . +scanoss-py scan --proxy "http://:" -o results.json . ``` \ No newline at end of file diff --git a/PACKAGE.md b/PACKAGE.md index 424a4e02..85b2c63b 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -110,5 +110,8 @@ Python 3.7 or higher. ## Source code The source for this package can be found [here](https://github.com/scanoss/scanoss.py). +## Documentation +For client usage help please look [here](https://github.com/scanoss/scanoss.py/blob/main/CLIENT_HELP.md). + ## Changelog Details of each release can be found [here](https://github.com/scanoss/scanoss.py/blob/main/CHANGELOG.md). diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 343d94b4..495b9a0a 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.0' +__version__ = '1.3.1' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 06df545e..67798de0 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -83,7 +83,9 @@ def setup_args() -> None: help='Scanning engine flags (1: disable snippet matching, 2 enable snippet ids, ' '4: disable dependencies, 8: disable licenses, 16: disable copyrights,' '32: disable vulnerabilities, 64: disable quality, 128: disable cryptography,' - '256: disable best match, 512: Report identified files)' + '256: disable best match only, 512: hide identified files, ' + '1024: enable download_url, 2048: enable GitHub full path, ' + '4096: disable extended server stats)' ) p_scan.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') p_scan.add_argument('--post-size', '-P', type=int, default=64, @@ -165,6 +167,15 @@ def setup_args() -> None: help='Display the location of Python CA Certs') p_c_loc.set_defaults(func=utils_certloc) + # Utils Sub-command: utils cert-download + p_c_dwnld = utils_sub.add_parser('cert-download', aliases=['cdl', 'cert-dl'], + description=f'Download Server SSL Cert: {__version__}', + help='Download the specified server\'s SSL PEM certificate') + p_c_dwnld.set_defaults(func=utils_cert_download) + p_c_dwnld.add_argument('--hostname', '-n', required=True, type=str, help='Server hostname to download cert from.' ) + p_c_dwnld.add_argument('--port', '-p', required=False, type=int, default=443, help='Server port number (default: 443).' ) + p_c_dwnld.add_argument('--output','-o', type=str, help='Output result file name (optional - default stdout).' ) + # Global command options for p in [p_scan]: p.add_argument('--key', '-k', type=str, @@ -187,7 +198,7 @@ def setup_args() -> None: ) p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') - for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc]: + for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -501,6 +512,61 @@ def utils_certloc(parser, args): import certifi print(f'CA Cert File: {certifi.where()}') +def utils_cert_download(parser, args): + """ + Run the "utils cert-download" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + import ssl + from urllib.parse import urlparse + import socket + import traceback + + file = sys.stdout + try: + if args.output: + file = open(args.output, 'w') + parsed_url = urlparse(args.hostname) + hostname = parsed_url.hostname or args.hostname # Use the parse hostname, or it None use the supplied one + port = int(parsed_url.port or args.port) # Use the parsed port, if not use the supplied one (default 443) + conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sock = context.wrap_socket(conn, server_hostname=hostname) + if not args.quiet or args.debug: + print_stderr(f'Attempting to download PEM certificate from {hostname}:{port} ...') + if args.debug: + print_stderr('Connecting to host...') + sock.connect((hostname, port)) + if args.debug: + print_stderr('Getting peer cert...') + peer_cert = sock.getpeercert(True) + if not peer_cert: + print_stderr(f'Error: Failed to download peer certificate data from {hostname}:{port}') + exit(1) + if args.debug: + print_stderr('Converting DER to PEM...') + cert_data = ssl.DER_cert_to_PEM_cert(peer_cert) + if not cert_data or cert_data == '': + print_stderr(f'Error: Failed to convert certificate data to PEM from {hostname}:{port}') + exit(1) + else: + print(cert_data.strip(), file=file) # Print the downloaded PEM certificate + except Exception as e: + print_stderr(f'ERROR: Exception ({e.__class__.__name__}) Downloading certificate from {hostname}:{port} - {e}.') + if args.debug: + traceback.print_exc() + exit(1) + else: + if args.output: + if args.debug: + print_stderr(f'Saved certificate to {args.output}') + file.close() + def main(): """ Run the ScanOSS CLI From b95bb87c973449185fa5cb9147b9ff68fa7c2858 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 22 Dec 2022 13:38:33 +0000 Subject: [PATCH 042/489] Avoid scanning empty WFPs --- src/scanoss/threadedscanning.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index 9d8223a9..adf1d8fd 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -125,7 +125,10 @@ def queue_add(self, wfp: str) -> None: Add requests to the queue :param wfp: WFP to add to queue """ - self.inputs.put(wfp) + if wfp is None or wfp == '': + self.print_stderr(f'Warning: empty WFP. Skipping from scan...') + else: + self.inputs.put(wfp) def get_queue_size(self) -> int: return self.inputs.qsize() @@ -190,6 +193,8 @@ def worker_post(self) -> None: wfp = self.inputs.get(timeout=5) self.print_trace(f'Processing input request ({current_thread})...') count = self.__count_files_in_wfp(wfp) + if wfp is None or wfp == '': + self.print_stderr(f'Warning: Empty WFP in request input: {wfp}') resp = self.scanapi.scan(wfp, scan_id=current_thread) if resp: self.output.put(resp) # Store the output response to later collection From ec5b39a74eef0a93a3e9bb954853f0c6f74a2139 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 22 Dec 2022 13:39:08 +0000 Subject: [PATCH 043/489] Warn about empty WFPs --- src/scanoss/winnowing.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index d8d3c267..8705dc16 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -237,12 +237,12 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')): return wfp # Initialize variables - gram = "" + gram = '' window = [] line = 1 last_hash = MAX_CRC32 last_line = 0 - output = "" + output = '' # Otherwise, recurse src_content and calculate Winnowing hashes for byte in contents: if byte == ASCII_LF: @@ -269,7 +269,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: crc = crc32c(min_hash.to_bytes(4, byteorder='little')) crc_hex = '{:08x}'.format(crc) if last_line != line: - if output: + if output != '': if self.size_limit and \ (len(wfp.encode("utf-8")) + len( output.encode("utf-8"))) > self.max_post_size: @@ -287,10 +287,14 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: window.pop(0) # Shift gram gram = gram[1:] - if output and ( - not self.size_limit or (len(wfp.encode("utf-8")) + len(output.encode("utf-8"))) < self.max_post_size): - wfp += output + '\n' + if output != '': + if not self.size_limit or (len(wfp.encode("utf-8")) + len(output.encode("utf-8"))) < self.max_post_size: + wfp += output + '\n' + else: + self.print_debug(f'Warning: skipping output in WFP for {file} - "{output}"') + if wfp is None or wfp == '': + self.print_stderr(f'Warning: No WFP content data for {file}') return wfp # From 800b865b7567456d3a202c3b7493ce1de244ebc8 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 22 Dec 2022 13:40:16 +0000 Subject: [PATCH 044/489] Fixed bug when limiting post size --- src/scanoss/scanner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 5d261265..41ab685f 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -375,11 +375,14 @@ def scan_folder(self, scan_dir: str) -> bool: if spinner: spinner.next() wfp = self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) + if wfp is None or wfp == '': + self.print_stderr(f'Warning: No WFP returned for {path}') wfp_list.append(wfp) file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode("utf-8")) - if (wfp_size + scan_size) >= self.max_post_size: + # If the wfp is bigger than the max post size and we already have something stored in the scan block, add it to the queue + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 scan_block = '' @@ -396,7 +399,7 @@ def scan_folder(self, scan_dir: str) -> bool: f'Warning: Some errors encounted while scanning. Results might be incomplete.') success = False # End for loop - if self.threaded_scan and scan_block: + if self.threaded_scan and scan_block != '': self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted if spinner: spinner.finish() @@ -566,7 +569,7 @@ def scan_file(self, file: str) -> bool: raise Exception(f"ERROR: Specified files does not exist or is not a file: {file}") self.print_debug(f'Fingerprinting {file}...') wfp = self.winnowing.wfp_for_file(file, file) - if wfp: + if wfp is not None and wfp != '': if self.threaded_scan: self.threaded_scan.queue_add(wfp) # Submit the WFP for scanning self.print_debug(f'Scanning {file}...') From a626217ca98e20243eae4126e6331519014e6f56 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 28 Dec 2022 19:45:40 +0000 Subject: [PATCH 045/489] added x-request-id support and better error response handling --- .gitignore | 1 + src/scanoss/__init__.py | 2 +- src/scanoss/scanossapi.py | 59 +++++++++++++++++++++++++++++---------- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index e8aa8281..01f36668 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ __pycache__ venv/ .idea src/scanoss/data/build_date.txt +bad*.txt diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 495b9a0a..ff44f7f4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.1' +__version__ = '1.3.2' diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 4de1340e..009b34fe 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -1,7 +1,7 @@ """ SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2022, SCANOSS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -118,6 +118,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): :param scan_id: ID of the scan being run (usually thread id) :return: JSON result object """ + request_id = str(uuid.uuid4()) form_data = {} if self.sbom: form_data['type'] = self.scan_type @@ -128,7 +129,9 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): form_data['flags'] = self.flags if context: form_data['context'] = context - scan_files = {'file': ("%s.wfp" % uuid.uuid1().hex, wfp)} + scan_files = {'file': ("%s.wfp" % request_id, wfp)} + headers = self.headers + headers['x-request-id'] = request_id # send a unique request id for each post r = None retry = 0 # Add some retry logic to cater for timeouts, etc. while retry <= 5: @@ -143,45 +146,55 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: if retry > 5: # Timed out 5 or more times, fail - self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data - {e}: {scan_files}') - raise Exception(f"ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for {self.url}") from e + self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data ({request_id}) - {e}: {scan_files}') + raise Exception(f"ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for" + f" {self.url}") from e else: self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...') time.sleep(5) except Exception as e: - self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}: {scan_files}') + self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) - {e}:' + f' {scan_files}') raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e else: - if not r: + if r is None: if retry > 5: # No response 5 or more times, fail - raise Exception(f"ERROR: The SCANOSS API request response object is empty for {self.url}") + self.save_bad_req_wfp(scan_files, request_id, scan_id) + raise Exception(f"ERROR: The SCANOSS API request ({request_id}) response object is empty " + f"for {self.url}") else: self.print_stderr(f'Warning: No response received from {self.url}. Retrying...') time.sleep(5) elif r.status_code >= 400: if retry > 5: # No response 5 or more times, fail + self.save_bad_req_wfp(scan_files, request_id, scan_id) raise Exception( - f"ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, {r.text}") + f"ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, " + f"{r.text.strip()}") else: - self.print_stderr(f'Warning: Error response code {r.status_code} from {self.url}. Retrying...') + self.save_bad_req_wfp(scan_files, request_id, scan_id) + self.print_stderr(f'Warning: Error response code {r.status_code} ({r.text.strip()}) from ' + f'{self.url}. Retrying...') time.sleep(5) else: break # Valid response, break out of the retry loop # End of while loop - if not r: + if r is None: + self.save_bad_req_wfp(scan_files, request_id, scan_id) raise Exception(f"ERROR: The SCANOSS API request response object is empty for {self.url}") try: - if 'xml' in self.scan_format: + if 'xml' in self.scan_format: # TODO remove XML parsing option? return r.text json_resp = r.json() return json_resp except (JSONDecodeError, Exception) as e: - self.print_stderr(f'ERROR: The SCANOSS API returned an invalid JSON ({e.__class__.__name__}): {e}') - ctime = int(time.time()) - bad_json_file = f'bad_json-{scan_id}-{ctime}.txt' if scan_id else f'bad_json-{ctime}.txt' + self.print_stderr(f'ERROR: The SCANOSS API returned an invalid JSON ' + f'({e.__class__.__name__} - {request_id}): {e}') + bad_json_file = f'bad_json-{scan_id}-{request_id}.txt' if scan_id else f'bad_json-{request_id}.txt' self.print_stderr(f'Ignoring result. Please look in "{bad_json_file}" for more details.') try: with open(bad_json_file, 'w') as f: + f.write(f"---Request ID Begin---\n{request_id}\n---Request ID End---\n") f.write(f"---WFP Begin---\n{scan_files}\n---WFP End---\n---Bad JSON Begin---\n") f.write(r.text) f.write("---Bad JSON End---\n") @@ -190,6 +203,24 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): f' {ee}') return None + def save_bad_req_wfp(self, scan_files, request_id, scan_id): + """ + Save the given WFP to a bad_request file + :param scan_files: WFP + :param request_id: request ID + :param scan_id: scan thread id (optional) + """ + bad_req_file = f'bad_request-{scan_id}-{request_id}.txt' if scan_id else f'bad_request-{request_id}.txt' + try: + self.print_stderr(f'No response object returned from API. Please look in "{bad_req_file}" for the ' + f'offending WFP.') + with open(bad_req_file, 'w') as f: + f.write(f"---Request ID Begin---\n{request_id}\n---Request ID End---\n") + f.write(f"---WFP Begin---\n{scan_files}\n---WFP End---\n") + except Exception as ee: + self.print_stderr(f'Warning: Issue writing bad request file - {bad_req_file} ({ee.__class__.__name__}):' + f' {ee}') + # # End of ScanossApi Class # From 046b4302c8d309a7a6bd66a1a10329fe226a20a4 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 28 Dec 2022 19:45:40 +0000 Subject: [PATCH 046/489] added x-request-id support and better error response handling --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 797fde06..b80b1afa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.2] - 2022-12-28 +### Added +- Added `x-request-id` to all scanning requests +- Added bad_request error log file to aid debug +### Fixed +- Fixed issue when fingerprinting large files with a small POST (`--post-size`) + ## [1.3.1] - 2022-12-07 ### Added - Added `utils cert-download` sub-command to help with the use of custom certificates @@ -166,3 +173,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.2.3]: https://github.com/scanoss/scanoss.py/compare/v1.2.2...v1.2.3 [1.3.0]: https://github.com/scanoss/scanoss.py/compare/v1.2.3...v1.3.0 [1.3.1]: https://github.com/scanoss/scanoss.py/compare/v1.3.0...v1.3.1 +[1.3.2]: https://github.com/scanoss/scanoss.py/compare/v1.3.1...v1.3.2 From 6d41306c0f7c04bb46f9f377b0e96b3af9e51bac Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 4 Jan 2023 16:05:16 +0000 Subject: [PATCH 047/489] added latest SPDX license definitions --- src/scanoss/data/spdx-exceptions.json | 545 +- src/scanoss/data/spdx-licenses.json | 7369 +++++++++++++------------ 2 files changed, 4045 insertions(+), 3869 deletions(-) diff --git a/src/scanoss/data/spdx-exceptions.json b/src/scanoss/data/spdx-exceptions.json index 2c7e07fe..35e36029 100644 --- a/src/scanoss/data/spdx-exceptions.json +++ b/src/scanoss/data/spdx-exceptions.json @@ -1,15 +1,28 @@ { - "licenseListVersion": "03c58ca", + "licenseListVersion": "166e97c", "exceptions": [ { - "reference": "./Fawkes-Runtime-exception.json", + "reference": "./389-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Fawkes-Runtime-exception.html", - "referenceNumber": 1, - "name": "Fawkes Runtime Exception", - "licenseExceptionId": "Fawkes-Runtime-exception", + "detailsUrl": "./389-exception.html", + "referenceNumber": 23, + "name": "389 Directory Server Exception", + "licenseExceptionId": "389-exception", "seeAlso": [ - "http://www.fawkesrobotics.org/about/license/" + "http://directory.fedoraproject.org/wiki/GPL_Exception_License_Text", + "https://web.archive.org/web/20080828121337/http://directory.fedoraproject.org/wiki/GPL_Exception_License_Text" + ] + }, + { + "reference": "./Autoconf-exception-2.0.json", + "isDeprecatedLicenseId": false, + "detailsUrl": "./Autoconf-exception-2.0.html", + "referenceNumber": 13, + "name": "Autoconf exception 2.0", + "licenseExceptionId": "Autoconf-exception-2.0", + "seeAlso": [ + "http://ac-archive.sourceforge.net/doc/copyright.html", + "http://ftp.gnu.org/gnu/autoconf/autoconf-2.59.tar.gz" ] }, { @@ -24,144 +37,143 @@ ] }, { - "reference": "./FLTK-exception.json", + "reference": "./Bison-exception-2.2.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./FLTK-exception.html", - "referenceNumber": 3, - "name": "FLTK exception", - "licenseExceptionId": "FLTK-exception", + "detailsUrl": "./Bison-exception-2.2.html", + "referenceNumber": 35, + "name": "Bison exception 2.2", + "licenseExceptionId": "Bison-exception-2.2", "seeAlso": [ - "http://www.fltk.org/COPYING.php" + "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" ] }, { - "reference": "./u-boot-exception-2.0.json", + "reference": "./Bootloader-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./u-boot-exception-2.0.html", - "referenceNumber": 4, - "name": "U-Boot exception 2.0", - "licenseExceptionId": "u-boot-exception-2.0", + "detailsUrl": "./Bootloader-exception.html", + "referenceNumber": 25, + "name": "Bootloader Distribution Exception", + "licenseExceptionId": "Bootloader-exception", "seeAlso": [ - "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003dLicenses/Exceptions" + "https://github.com/pyinstaller/pyinstaller/blob/develop/COPYING.txt" ] }, { - "reference": "./CLISP-exception-2.0.json", + "reference": "./Classpath-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./CLISP-exception-2.0.html", - "referenceNumber": 5, - "name": "CLISP exception 2.0", - "licenseExceptionId": "CLISP-exception-2.0", + "detailsUrl": "./Classpath-exception-2.0.html", + "referenceNumber": 27, + "name": "Classpath exception 2.0", + "licenseExceptionId": "Classpath-exception-2.0", "seeAlso": [ - "http://sourceforge.net/p/clisp/clisp/ci/default/tree/COPYRIGHT" + "http://www.gnu.org/software/classpath/license.html", + "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception" ] }, { - "reference": "./WxWindows-exception-3.1.json", + "reference": "./CLISP-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./WxWindows-exception-3.1.html", - "referenceNumber": 6, - "name": "WxWindows Library Exception 3.1", - "licenseExceptionId": "WxWindows-exception-3.1", + "detailsUrl": "./CLISP-exception-2.0.html", + "referenceNumber": 20, + "name": "CLISP exception 2.0", + "licenseExceptionId": "CLISP-exception-2.0", "seeAlso": [ - "http://www.opensource.org/licenses/WXwindows" + "http://sourceforge.net/p/clisp/clisp/ci/default/tree/COPYRIGHT" ] }, { - "reference": "./389-exception.json", + "reference": "./DigiRule-FOSS-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./389-exception.html", - "referenceNumber": 7, - "name": "389 Directory Server Exception", - "licenseExceptionId": "389-exception", + "detailsUrl": "./DigiRule-FOSS-exception.html", + "referenceNumber": 34, + "name": "DigiRule FOSS License Exception", + "licenseExceptionId": "DigiRule-FOSS-exception", "seeAlso": [ - "http://directory.fedoraproject.org/wiki/GPL_Exception_License_Text", - "https://web.archive.org/web/20080828121337/http://directory.fedoraproject.org/wiki/GPL_Exception_License_Text" + "http://www.digirulesolutions.com/drupal/foss" ] }, { - "reference": "./SHL-2.0.json", + "reference": "./eCos-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./SHL-2.0.html", - "referenceNumber": 8, - "name": "Solderpad Hardware License v2.0", - "licenseExceptionId": "SHL-2.0", + "detailsUrl": "./eCos-exception-2.0.html", + "referenceNumber": 44, + "name": "eCos exception 2.0", + "licenseExceptionId": "eCos-exception-2.0", "seeAlso": [ - "https://solderpad.org/licenses/SHL-2.0/" + "http://ecos.sourceware.org/license-overview.html" ] }, { - "reference": "./DigiRule-FOSS-exception.json", + "reference": "./Fawkes-Runtime-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./DigiRule-FOSS-exception.html", - "referenceNumber": 9, - "name": "DigiRule FOSS License Exception", - "licenseExceptionId": "DigiRule-FOSS-exception", + "detailsUrl": "./Fawkes-Runtime-exception.html", + "referenceNumber": 7, + "name": "Fawkes Runtime Exception", + "licenseExceptionId": "Fawkes-Runtime-exception", "seeAlso": [ - "http://www.digirulesolutions.com/drupal/foss" + "http://www.fawkesrobotics.org/about/license/" ] }, { - "reference": "./KiCad-libraries-exception.json", + "reference": "./FLTK-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./KiCad-libraries-exception.html", - "referenceNumber": 10, - "name": "KiCad Libraries Exception", - "licenseExceptionId": "KiCad-libraries-exception", + "detailsUrl": "./FLTK-exception.html", + "referenceNumber": 43, + "name": "FLTK exception", + "licenseExceptionId": "FLTK-exception", "seeAlso": [ - "https://www.kicad.org/libraries/license/" + "http://www.fltk.org/COPYING.php" ] }, { - "reference": "./GStreamer-exception-2005.json", + "reference": "./Font-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GStreamer-exception-2005.html", - "referenceNumber": 11, - "name": "GStreamer Exception (2005)", - "licenseExceptionId": "GStreamer-exception-2005", + "detailsUrl": "./Font-exception-2.0.html", + "referenceNumber": 3, + "name": "Font exception 2.0", + "licenseExceptionId": "Font-exception-2.0", "seeAlso": [ - "https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language\u003dc#licensing-of-applications-using-gstreamer" + "http://www.gnu.org/licenses/gpl-faq.html#FontException" ] }, { - "reference": "./Qwt-exception-1.0.json", + "reference": "./freertos-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Qwt-exception-1.0.html", - "referenceNumber": 12, - "name": "Qwt exception 1.0", - "licenseExceptionId": "Qwt-exception-1.0", + "detailsUrl": "./freertos-exception-2.0.html", + "referenceNumber": 32, + "name": "FreeRTOS Exception 2.0", + "licenseExceptionId": "freertos-exception-2.0", "seeAlso": [ - "http://qwt.sourceforge.net/qwtlicense.html" + "https://web.archive.org/web/20060809182744/http://www.freertos.org/a00114.html" ] }, { - "reference": "./GPL-3.0-linking-source-exception.json", + "reference": "./GCC-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GPL-3.0-linking-source-exception.html", - "referenceNumber": 13, - "name": "GPL-3.0 Linking Exception (with Corresponding Source)", - "licenseExceptionId": "GPL-3.0-linking-source-exception", + "detailsUrl": "./GCC-exception-2.0.html", + "referenceNumber": 19, + "name": "GCC Runtime Library exception 2.0", + "licenseExceptionId": "GCC-exception-2.0", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs", - "https://github.com/mirror/wget/blob/master/src/http.c#L20" + "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" ] }, { - "reference": "./OCaml-LGPL-linking-exception.json", + "reference": "./GCC-exception-3.1.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./OCaml-LGPL-linking-exception.html", - "referenceNumber": 14, - "name": "OCaml LGPL Linking Exception", - "licenseExceptionId": "OCaml-LGPL-linking-exception", + "detailsUrl": "./GCC-exception-3.1.html", + "referenceNumber": 31, + "name": "GCC Runtime Library exception 3.1", + "licenseExceptionId": "GCC-exception-3.1", "seeAlso": [ - "https://caml.inria.fr/ocaml/license.en.html" + "http://www.gnu.org/licenses/gcc-exception-3.1.html" ] }, { "reference": "./gnu-javamail-exception.json", "isDeprecatedLicenseId": false, "detailsUrl": "./gnu-javamail-exception.html", - "referenceNumber": 15, + "referenceNumber": 8, "name": "GNU JavaMail exception", "licenseExceptionId": "gnu-javamail-exception", "seeAlso": [ @@ -169,332 +181,331 @@ ] }, { - "reference": "./Libtool-exception.json", + "reference": "./GPL-3.0-linking-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Libtool-exception.html", - "referenceNumber": 16, - "name": "Libtool Exception", - "licenseExceptionId": "Libtool-exception", + "detailsUrl": "./GPL-3.0-linking-exception.html", + "referenceNumber": 18, + "name": "GPL-3.0 Linking Exception", + "licenseExceptionId": "GPL-3.0-linking-exception", "seeAlso": [ - "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4" + "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs" ] }, { - "reference": "./LGPL-3.0-linking-exception.json", + "reference": "./GPL-3.0-linking-source-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./LGPL-3.0-linking-exception.html", - "referenceNumber": 17, - "name": "LGPL-3.0 Linking Exception", - "licenseExceptionId": "LGPL-3.0-linking-exception", + "detailsUrl": "./GPL-3.0-linking-source-exception.html", + "referenceNumber": 30, + "name": "GPL-3.0 Linking Exception (with Corresponding Source)", + "licenseExceptionId": "GPL-3.0-linking-source-exception", "seeAlso": [ - "https://raw.githubusercontent.com/go-xmlpath/xmlpath/v2/LICENSE", - "https://github.com/goamz/goamz/blob/master/LICENSE", - "https://github.com/juju/errors/blob/master/LICENSE" + "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs", + "https://github.com/mirror/wget/blob/master/src/http.c#L20" ] }, { - "reference": "./mif-exception.json", + "reference": "./GPL-CC-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./mif-exception.html", - "referenceNumber": 18, - "name": "Macros and Inline Functions Exception", - "licenseExceptionId": "mif-exception", + "detailsUrl": "./GPL-CC-1.0.html", + "referenceNumber": 45, + "name": "GPL Cooperation Commitment 1.0", + "licenseExceptionId": "GPL-CC-1.0", "seeAlso": [ - "http://www.scs.stanford.edu/histar/src/lib/cppsup/exception", - "http://dev.bertos.org/doxygen/", - "https://www.threadingbuildingblocks.org/licensing" + "https://github.com/gplcc/gplcc/blob/master/Project/COMMITMENT", + "https://gplcc.github.io/gplcc/Project/README-PROJECT.html" ] }, { - "reference": "./openvpn-openssl-exception.json", + "reference": "./GStreamer-exception-2005.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./openvpn-openssl-exception.html", - "referenceNumber": 19, - "name": "OpenVPN OpenSSL Exception", - "licenseExceptionId": "openvpn-openssl-exception", + "detailsUrl": "./GStreamer-exception-2005.html", + "referenceNumber": 28, + "name": "GStreamer Exception (2005)", + "licenseExceptionId": "GStreamer-exception-2005", "seeAlso": [ - "http://openvpn.net/index.php/license.html" + "https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language\u003dc#licensing-of-applications-using-gstreamer" ] }, { - "reference": "./GCC-exception-2.0.json", + "reference": "./GStreamer-exception-2008.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GCC-exception-2.0.html", - "referenceNumber": 20, - "name": "GCC Runtime Library exception 2.0", - "licenseExceptionId": "GCC-exception-2.0", + "detailsUrl": "./GStreamer-exception-2008.html", + "referenceNumber": 10, + "name": "GStreamer Exception (2008)", + "licenseExceptionId": "GStreamer-exception-2008", "seeAlso": [ - "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" + "https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language\u003dc#licensing-of-applications-using-gstreamer" ] }, { - "reference": "./Bison-exception-2.2.json", + "reference": "./i2p-gpl-java-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Bison-exception-2.2.html", - "referenceNumber": 21, - "name": "Bison exception 2.2", - "licenseExceptionId": "Bison-exception-2.2", + "detailsUrl": "./i2p-gpl-java-exception.html", + "referenceNumber": 42, + "name": "i2p GPL+Java Exception", + "licenseExceptionId": "i2p-gpl-java-exception", "seeAlso": [ - "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" + "http://geti2p.net/en/get-involved/develop/licenses#java_exception" ] }, { - "reference": "./eCos-exception-2.0.json", + "reference": "./KiCad-libraries-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./eCos-exception-2.0.html", - "referenceNumber": 22, - "name": "eCos exception 2.0", - "licenseExceptionId": "eCos-exception-2.0", + "detailsUrl": "./KiCad-libraries-exception.html", + "referenceNumber": 38, + "name": "KiCad Libraries Exception", + "licenseExceptionId": "KiCad-libraries-exception", "seeAlso": [ - "http://ecos.sourceware.org/license-overview.html" + "https://www.kicad.org/libraries/license/" ] }, { - "reference": "./LLVM-exception.json", + "reference": "./LGPL-3.0-linking-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./LLVM-exception.html", - "referenceNumber": 23, - "name": "LLVM Exception", - "licenseExceptionId": "LLVM-exception", + "detailsUrl": "./LGPL-3.0-linking-exception.html", + "referenceNumber": 16, + "name": "LGPL-3.0 Linking Exception", + "licenseExceptionId": "LGPL-3.0-linking-exception", "seeAlso": [ - "http://llvm.org/foundation/relicensing/LICENSE.txt" + "https://raw.githubusercontent.com/go-xmlpath/xmlpath/v2/LICENSE", + "https://github.com/goamz/goamz/blob/master/LICENSE", + "https://github.com/juju/errors/blob/master/LICENSE" ] }, { - "reference": "./i2p-gpl-java-exception.json", + "reference": "./Libtool-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./i2p-gpl-java-exception.html", - "referenceNumber": 24, - "name": "i2p GPL+Java Exception", - "licenseExceptionId": "i2p-gpl-java-exception", + "detailsUrl": "./Libtool-exception.html", + "referenceNumber": 4, + "name": "Libtool Exception", + "licenseExceptionId": "Libtool-exception", "seeAlso": [ - "http://geti2p.net/en/get-involved/develop/licenses#java_exception" + "http://git.savannah.gnu.org/cgit/libtool.git/tree/m4/libtool.m4" ] }, { - "reference": "./GPL-CC-1.0.json", + "reference": "./Linux-syscall-note.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GPL-CC-1.0.html", - "referenceNumber": 25, - "name": "GPL Cooperation Commitment 1.0", - "licenseExceptionId": "GPL-CC-1.0", + "detailsUrl": "./Linux-syscall-note.html", + "referenceNumber": 6, + "name": "Linux Syscall Note", + "licenseExceptionId": "Linux-syscall-note", "seeAlso": [ - "https://github.com/gplcc/gplcc/blob/master/Project/COMMITMENT", - "https://gplcc.github.io/gplcc/Project/README-PROJECT.html" + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/COPYING" ] }, { - "reference": "./Classpath-exception-2.0.json", + "reference": "./LLVM-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Classpath-exception-2.0.html", - "referenceNumber": 26, - "name": "Classpath exception 2.0", - "licenseExceptionId": "Classpath-exception-2.0", + "detailsUrl": "./LLVM-exception.html", + "referenceNumber": 21, + "name": "LLVM Exception", + "licenseExceptionId": "LLVM-exception", "seeAlso": [ - "http://www.gnu.org/software/classpath/license.html", - "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception" + "http://llvm.org/foundation/relicensing/LICENSE.txt" ] }, { - "reference": "./freertos-exception-2.0.json", + "reference": "./LZMA-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./freertos-exception-2.0.html", - "referenceNumber": 27, - "name": "FreeRTOS Exception 2.0", - "licenseExceptionId": "freertos-exception-2.0", + "detailsUrl": "./LZMA-exception.html", + "referenceNumber": 11, + "name": "LZMA exception", + "licenseExceptionId": "LZMA-exception", "seeAlso": [ - "https://web.archive.org/web/20060809182744/http://www.freertos.org/a00114.html" + "http://nsis.sourceforge.net/Docs/AppendixI.html#I.6" ] }, { - "reference": "./Qt-GPL-exception-1.0.json", + "reference": "./mif-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Qt-GPL-exception-1.0.html", - "referenceNumber": 28, - "name": "Qt GPL exception 1.0", - "licenseExceptionId": "Qt-GPL-exception-1.0", + "detailsUrl": "./mif-exception.html", + "referenceNumber": 33, + "name": "Macros and Inline Functions Exception", + "licenseExceptionId": "mif-exception", "seeAlso": [ - "http://code.qt.io/cgit/qt/qtbase.git/tree/LICENSE.GPL3-EXCEPT" + "http://www.scs.stanford.edu/histar/src/lib/cppsup/exception", + "http://dev.bertos.org/doxygen/", + "https://www.threadingbuildingblocks.org/licensing" ] }, { - "reference": "./GStreamer-exception-2008.json", - "isDeprecatedLicenseId": false, - "detailsUrl": "./GStreamer-exception-2008.html", - "referenceNumber": 29, - "name": "GStreamer Exception (2008)", - "licenseExceptionId": "GStreamer-exception-2008", + "reference": "./Nokia-Qt-exception-1.1.json", + "isDeprecatedLicenseId": true, + "detailsUrl": "./Nokia-Qt-exception-1.1.html", + "referenceNumber": 17, + "name": "Nokia Qt LGPL exception 1.1", + "licenseExceptionId": "Nokia-Qt-exception-1.1", "seeAlso": [ - "https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language\u003dc#licensing-of-applications-using-gstreamer" + "https://www.keepassx.org/dev/projects/keepassx/repository/revisions/b8dfb9cc4d5133e0f09cd7533d15a4f1c19a40f2/entry/LICENSE.NOKIA-LGPL-EXCEPTION" ] }, { - "reference": "./PS-or-PDF-font-exception-20170817.json", + "reference": "./OCaml-LGPL-linking-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./PS-or-PDF-font-exception-20170817.html", - "referenceNumber": 30, - "name": "PS/PDF font exception (2017-08-17)", - "licenseExceptionId": "PS-or-PDF-font-exception-20170817", + "detailsUrl": "./OCaml-LGPL-linking-exception.html", + "referenceNumber": 12, + "name": "OCaml LGPL Linking Exception", + "licenseExceptionId": "OCaml-LGPL-linking-exception", "seeAlso": [ - "https://github.com/ArtifexSoftware/urw-base35-fonts/blob/65962e27febc3883a17e651cdb23e783668c996f/LICENSE" + "https://caml.inria.fr/ocaml/license.en.html" ] }, { - "reference": "./Universal-FOSS-exception-1.0.json", + "reference": "./OCCT-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Universal-FOSS-exception-1.0.html", - "referenceNumber": 31, - "name": "Universal FOSS Exception, Version 1.0", - "licenseExceptionId": "Universal-FOSS-exception-1.0", + "detailsUrl": "./OCCT-exception-1.0.html", + "referenceNumber": 1, + "name": "Open CASCADE Exception 1.0", + "licenseExceptionId": "OCCT-exception-1.0", "seeAlso": [ - "https://oss.oracle.com/licenses/universal-foss-exception/" + "http://www.opencascade.com/content/licensing" ] }, { - "reference": "./Nokia-Qt-exception-1.1.json", - "isDeprecatedLicenseId": true, - "detailsUrl": "./Nokia-Qt-exception-1.1.html", - "referenceNumber": 32, - "name": "Nokia Qt LGPL exception 1.1", - "licenseExceptionId": "Nokia-Qt-exception-1.1", + "reference": "./OpenJDK-assembly-exception-1.0.json", + "isDeprecatedLicenseId": false, + "detailsUrl": "./OpenJDK-assembly-exception-1.0.html", + "referenceNumber": 5, + "name": "OpenJDK Assembly exception 1.0", + "licenseExceptionId": "OpenJDK-assembly-exception-1.0", "seeAlso": [ - "https://www.keepassx.org/dev/projects/keepassx/repository/revisions/b8dfb9cc4d5133e0f09cd7533d15a4f1c19a40f2/entry/LICENSE.NOKIA-LGPL-EXCEPTION" + "http://openjdk.java.net/legal/assembly-exception.html" ] }, { - "reference": "./SHL-2.1.json", + "reference": "./openvpn-openssl-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./SHL-2.1.html", - "referenceNumber": 33, - "name": "Solderpad Hardware License v2.1", - "licenseExceptionId": "SHL-2.1", + "detailsUrl": "./openvpn-openssl-exception.html", + "referenceNumber": 37, + "name": "OpenVPN OpenSSL Exception", + "licenseExceptionId": "openvpn-openssl-exception", "seeAlso": [ - "https://solderpad.org/licenses/SHL-2.1/" + "http://openvpn.net/index.php/license.html" ] }, { - "reference": "./Font-exception-2.0.json", + "reference": "./PS-or-PDF-font-exception-20170817.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Font-exception-2.0.html", - "referenceNumber": 34, - "name": "Font exception 2.0", - "licenseExceptionId": "Font-exception-2.0", + "detailsUrl": "./PS-or-PDF-font-exception-20170817.html", + "referenceNumber": 36, + "name": "PS/PDF font exception (2017-08-17)", + "licenseExceptionId": "PS-or-PDF-font-exception-20170817", "seeAlso": [ - "http://www.gnu.org/licenses/gpl-faq.html#FontException" + "https://github.com/ArtifexSoftware/urw-base35-fonts/blob/65962e27febc3883a17e651cdb23e783668c996f/LICENSE" ] }, { - "reference": "./Bootloader-exception.json", + "reference": "./Qt-GPL-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Bootloader-exception.html", - "referenceNumber": 35, - "name": "Bootloader Distribution Exception", - "licenseExceptionId": "Bootloader-exception", + "detailsUrl": "./Qt-GPL-exception-1.0.html", + "referenceNumber": 40, + "name": "Qt GPL exception 1.0", + "licenseExceptionId": "Qt-GPL-exception-1.0", "seeAlso": [ - "https://github.com/pyinstaller/pyinstaller/blob/develop/COPYING.txt" + "http://code.qt.io/cgit/qt/qtbase.git/tree/LICENSE.GPL3-EXCEPT" ] }, { - "reference": "./OpenJDK-assembly-exception-1.0.json", + "reference": "./Qt-LGPL-exception-1.1.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./OpenJDK-assembly-exception-1.0.html", - "referenceNumber": 36, - "name": "OpenJDK Assembly exception 1.0", - "licenseExceptionId": "OpenJDK-assembly-exception-1.0", + "detailsUrl": "./Qt-LGPL-exception-1.1.html", + "referenceNumber": 29, + "name": "Qt LGPL exception 1.1", + "licenseExceptionId": "Qt-LGPL-exception-1.1", "seeAlso": [ - "http://openjdk.java.net/legal/assembly-exception.html" + "http://code.qt.io/cgit/qt/qtbase.git/tree/LGPL_EXCEPTION.txt" ] }, { - "reference": "./Swift-exception.json", + "reference": "./Qwt-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Swift-exception.html", - "referenceNumber": 37, - "name": "Swift Exception", - "licenseExceptionId": "Swift-exception", + "detailsUrl": "./Qwt-exception-1.0.html", + "referenceNumber": 41, + "name": "Qwt exception 1.0", + "licenseExceptionId": "Qwt-exception-1.0", "seeAlso": [ - "https://swift.org/LICENSE.txt", - "https://github.com/apple/swift-package-manager/blob/7ab2275f447a5eb37497ed63a9340f8a6d1e488b/LICENSE.txt#L205" + "http://qwt.sourceforge.net/qwtlicense.html" ] }, { - "reference": "./GCC-exception-3.1.json", + "reference": "./SHL-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GCC-exception-3.1.html", - "referenceNumber": 38, - "name": "GCC Runtime Library exception 3.1", - "licenseExceptionId": "GCC-exception-3.1", + "detailsUrl": "./SHL-2.0.html", + "referenceNumber": 22, + "name": "Solderpad Hardware License v2.0", + "licenseExceptionId": "SHL-2.0", "seeAlso": [ - "http://www.gnu.org/licenses/gcc-exception-3.1.html" + "https://solderpad.org/licenses/SHL-2.0/" ] }, { - "reference": "./Linux-syscall-note.json", + "reference": "./SHL-2.1.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Linux-syscall-note.html", - "referenceNumber": 39, - "name": "Linux Syscall Note", - "licenseExceptionId": "Linux-syscall-note", + "detailsUrl": "./SHL-2.1.html", + "referenceNumber": 24, + "name": "Solderpad Hardware License v2.1", + "licenseExceptionId": "SHL-2.1", "seeAlso": [ - "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/COPYING" + "https://solderpad.org/licenses/SHL-2.1/" ] }, { - "reference": "./Autoconf-exception-2.0.json", + "reference": "./Swift-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Autoconf-exception-2.0.html", - "referenceNumber": 40, - "name": "Autoconf exception 2.0", - "licenseExceptionId": "Autoconf-exception-2.0", + "detailsUrl": "./Swift-exception.html", + "referenceNumber": 15, + "name": "Swift Exception", + "licenseExceptionId": "Swift-exception", "seeAlso": [ - "http://ac-archive.sourceforge.net/doc/copyright.html", - "http://ftp.gnu.org/gnu/autoconf/autoconf-2.59.tar.gz" + "https://swift.org/LICENSE.txt", + "https://github.com/apple/swift-package-manager/blob/7ab2275f447a5eb37497ed63a9340f8a6d1e488b/LICENSE.txt#L205" ] }, { - "reference": "./GPL-3.0-linking-exception.json", + "reference": "./u-boot-exception-2.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./GPL-3.0-linking-exception.html", - "referenceNumber": 41, - "name": "GPL-3.0 Linking Exception", - "licenseExceptionId": "GPL-3.0-linking-exception", + "detailsUrl": "./u-boot-exception-2.0.html", + "referenceNumber": 14, + "name": "U-Boot exception 2.0", + "licenseExceptionId": "u-boot-exception-2.0", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-faq.en.html#GPLIncompatibleLibs" + "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003dLicenses/Exceptions" ] }, { - "reference": "./Qt-LGPL-exception-1.1.json", + "reference": "./Universal-FOSS-exception-1.0.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./Qt-LGPL-exception-1.1.html", - "referenceNumber": 42, - "name": "Qt LGPL exception 1.1", - "licenseExceptionId": "Qt-LGPL-exception-1.1", + "detailsUrl": "./Universal-FOSS-exception-1.0.html", + "referenceNumber": 39, + "name": "Universal FOSS Exception, Version 1.0", + "licenseExceptionId": "Universal-FOSS-exception-1.0", "seeAlso": [ - "http://code.qt.io/cgit/qt/qtbase.git/tree/LGPL_EXCEPTION.txt" + "https://oss.oracle.com/licenses/universal-foss-exception/" ] }, { - "reference": "./OCCT-exception-1.0.json", + "reference": "./WxWindows-exception-3.1.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./OCCT-exception-1.0.html", - "referenceNumber": 43, - "name": "Open CASCADE Exception 1.0", - "licenseExceptionId": "OCCT-exception-1.0", + "detailsUrl": "./WxWindows-exception-3.1.html", + "referenceNumber": 9, + "name": "WxWindows Library Exception 3.1", + "licenseExceptionId": "WxWindows-exception-3.1", "seeAlso": [ - "http://www.opencascade.com/content/licensing" + "http://www.opensource.org/licenses/WXwindows" ] }, { - "reference": "./LZMA-exception.json", + "reference": "./x11vnc-openssl-exception.json", "isDeprecatedLicenseId": false, - "detailsUrl": "./LZMA-exception.html", - "referenceNumber": 44, - "name": "LZMA exception", - "licenseExceptionId": "LZMA-exception", + "detailsUrl": "./x11vnc-openssl-exception.html", + "referenceNumber": 26, + "name": "x11vnc OpenSSL Exception", + "licenseExceptionId": "x11vnc-openssl-exception", "seeAlso": [ - "http://nsis.sourceforge.net/Docs/AppendixI.html#I.6" + "https://github.com/LibVNC/x11vnc/blob/master/src/8to24.c#L22" ] } ], - "releaseDate": "2022-10-07" + "releaseDate": "2023-01-02" } \ No newline at end of file diff --git a/src/scanoss/data/spdx-licenses.json b/src/scanoss/data/spdx-licenses.json index 7e342897..b92b9b46 100644 --- a/src/scanoss/data/spdx-licenses.json +++ b/src/scanoss/data/spdx-licenses.json @@ -1,1544 +1,1517 @@ { - "licenseListVersion": "03c58ca", + "licenseListVersion": "166e97c", "licenses": [ { - "reference": "https://spdx.org/licenses/xpp.html", + "reference": "https://spdx.org/licenses/0BSD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/xpp.json", - "referenceNumber": 0, - "name": "XPP License", - "licenseId": "xpp", + "detailsUrl": "https://spdx.org/licenses/0BSD.json", + "referenceNumber": 413, + "name": "BSD Zero Clause License", + "licenseId": "0BSD", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/xpp" + "http://landley.net/toybox/license.html", + "https://opensource.org/licenses/0BSD" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-only.html", + "reference": "https://spdx.org/licenses/AAL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-only.json", - "referenceNumber": 1, - "name": "GNU Free Documentation License v1.1 only", - "licenseId": "GFDL-1.1-only", + "detailsUrl": "https://spdx.org/licenses/AAL.json", + "referenceNumber": 192, + "name": "Attribution Assurance License", + "licenseId": "AAL", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://opensource.org/licenses/attribution" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.json", - "referenceNumber": 2, - "name": "GNU General Public License v2.0 w/Autoconf exception", - "licenseId": "GPL-2.0-with-autoconf-exception", + "reference": "https://spdx.org/licenses/Abstyles.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Abstyles.json", + "referenceNumber": 326, + "name": "Abstyles License", + "licenseId": "Abstyles", "seeAlso": [ - "http://ac-archive.sourceforge.net/doc/copyright.html" + "https://fedoraproject.org/wiki/Licensing/Abstyles" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/copyleft-next-0.3.1.html", + "reference": "https://spdx.org/licenses/Adobe-2006.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.1.json", - "referenceNumber": 3, - "name": "copyleft-next 0.3.1", - "licenseId": "copyleft-next-0.3.1", + "detailsUrl": "https://spdx.org/licenses/Adobe-2006.json", + "referenceNumber": 6, + "name": "Adobe Systems Incorporated Source Code License Agreement", + "licenseId": "Adobe-2006", "seeAlso": [ - "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.1" + "https://fedoraproject.org/wiki/Licensing/AdobeLicense" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-2.5.html", + "reference": "https://spdx.org/licenses/Adobe-Glyph.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.5.json", - "referenceNumber": 4, - "name": "Creative Commons Attribution Non Commercial 2.5 Generic", - "licenseId": "CC-BY-NC-2.5", + "detailsUrl": "https://spdx.org/licenses/Adobe-Glyph.json", + "referenceNumber": 330, + "name": "Adobe Glyph List License", + "licenseId": "Adobe-Glyph", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/2.5/legalcode" + "https://fedoraproject.org/wiki/Licensing/MIT#AdobeGlyph" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.html", + "reference": "https://spdx.org/licenses/ADSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json", - "referenceNumber": 5, - "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", - "licenseId": "CC-BY-NC-SA-3.0-IGO", + "detailsUrl": "https://spdx.org/licenses/ADSL.json", + "referenceNumber": 133, + "name": "Amazon Digital Services License", + "licenseId": "ADSL", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/3.0/igo/legalcode" + "https://fedoraproject.org/wiki/Licensing/AmazonDigitalServicesLicense" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/QPL-1.0.html", + "reference": "https://spdx.org/licenses/AFL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/QPL-1.0.json", - "referenceNumber": 6, - "name": "Q Public License 1.0", - "licenseId": "QPL-1.0", + "detailsUrl": "https://spdx.org/licenses/AFL-1.1.json", + "referenceNumber": 143, + "name": "Academic Free License v1.1", + "licenseId": "AFL-1.1", "seeAlso": [ - "http://doc.qt.nokia.com/3.3/license.html", - "https://opensource.org/licenses/QPL-1.0", - "https://doc.qt.io/archives/3.3/license.html" + "http://opensource.linux-mirror.org/licenses/afl-1.1.txt", + "http://wayback.archive.org/web/20021004124254/http://www.opensource.org/licenses/academic.php" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/ErlPL-1.1.html", + "reference": "https://spdx.org/licenses/AFL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ErlPL-1.1.json", - "referenceNumber": 7, - "name": "Erlang Public License v1.1", - "licenseId": "ErlPL-1.1", + "detailsUrl": "https://spdx.org/licenses/AFL-1.2.json", + "referenceNumber": 411, + "name": "Academic Free License v1.2", + "licenseId": "AFL-1.2", "seeAlso": [ - "http://www.erlang.org/EPLICENSE" + "http://opensource.linux-mirror.org/licenses/afl-1.2.txt", + "http://wayback.archive.org/web/20021204204652/http://www.opensource.org/licenses/academic.php" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Wsuipa.html", + "reference": "https://spdx.org/licenses/AFL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Wsuipa.json", - "referenceNumber": 8, - "name": "Wsuipa License", - "licenseId": "Wsuipa", + "detailsUrl": "https://spdx.org/licenses/AFL-2.0.json", + "referenceNumber": 263, + "name": "Academic Free License v2.0", + "licenseId": "AFL-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Wsuipa" + "http://wayback.archive.org/web/20060924134533/http://www.opensource.org/licenses/afl-2.0.txt" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Naumen.html", + "reference": "https://spdx.org/licenses/AFL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Naumen.json", - "referenceNumber": 9, - "name": "Naumen Public License", - "licenseId": "Naumen", + "detailsUrl": "https://spdx.org/licenses/AFL-2.1.json", + "referenceNumber": 363, + "name": "Academic Free License v2.1", + "licenseId": "AFL-2.1", "seeAlso": [ - "https://opensource.org/licenses/Naumen" + "http://opensource.linux-mirror.org/licenses/afl-2.1.txt" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/FSFUL.html", + "reference": "https://spdx.org/licenses/AFL-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FSFUL.json", - "referenceNumber": 10, - "name": "FSF Unlimited License", - "licenseId": "FSFUL", + "detailsUrl": "https://spdx.org/licenses/AFL-3.0.json", + "referenceNumber": 147, + "name": "Academic Free License v3.0", + "licenseId": "AFL-3.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License" + "http://www.rosenlaw.com/AFL3.0.htm", + "https://opensource.org/licenses/afl-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/AGPL-3.0-or-later.html", + "reference": "https://spdx.org/licenses/Afmparse.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-or-later.json", - "referenceNumber": 11, - "name": "GNU Affero General Public License v3.0 or later", - "licenseId": "AGPL-3.0-or-later", + "detailsUrl": "https://spdx.org/licenses/Afmparse.json", + "referenceNumber": 468, + "name": "Afmparse License", + "licenseId": "Afmparse", "seeAlso": [ - "https://www.gnu.org/licenses/agpl.txt", - "https://opensource.org/licenses/AGPL-3.0" + "https://fedoraproject.org/wiki/Licensing/Afmparse" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-3.0-or-later.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-or-later.json", - "referenceNumber": 12, - "name": "GNU Lesser General Public License v3.0 or later", - "licenseId": "LGPL-3.0-or-later", + "reference": "https://spdx.org/licenses/AGPL-1.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/AGPL-1.0.json", + "referenceNumber": 444, + "name": "Affero General Public License v1.0", + "licenseId": "AGPL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", - "https://opensource.org/licenses/LGPL-3.0" + "http://www.affero.org/oagpl.html" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NTP-0.html", + "reference": "https://spdx.org/licenses/AGPL-1.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NTP-0.json", - "referenceNumber": 13, - "name": "NTP No Attribution", - "licenseId": "NTP-0", + "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-only.json", + "referenceNumber": 458, + "name": "Affero General Public License v1.0 only", + "licenseId": "AGPL-1.0-only", "seeAlso": [ - "https://github.com/tytso/e2fsprogs/blob/master/lib/et/et_name.c" + "http://www.affero.org/oagpl.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NAIST-2003.html", + "reference": "https://spdx.org/licenses/AGPL-1.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NAIST-2003.json", - "referenceNumber": 14, - "name": "Nara Institute of Science and Technology License (2003)", - "licenseId": "NAIST-2003", + "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-or-later.json", + "referenceNumber": 97, + "name": "Affero General Public License v1.0 or later", + "licenseId": "AGPL-1.0-or-later", "seeAlso": [ - "https://enterprise.dejacode.com/licenses/public/naist-2003/#license-text", - "https://github.com/nodejs/node/blob/4a19cc8947b1bba2b2d27816ec3d0edf9b28e503/LICENSE#L343" + "http://www.affero.org/oagpl.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Newsletr.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Newsletr.json", - "referenceNumber": 15, - "name": "Newsletr License", - "licenseId": "Newsletr", + "reference": "https://spdx.org/licenses/AGPL-3.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/AGPL-3.0.json", + "referenceNumber": 385, + "name": "GNU Affero General Public License v3.0", + "licenseId": "AGPL-3.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Newsletr" + "https://www.gnu.org/licenses/agpl.txt", + "https://opensource.org/licenses/AGPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MPL-2.0.html", + "reference": "https://spdx.org/licenses/AGPL-3.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-2.0.json", - "referenceNumber": 16, - "name": "Mozilla Public License 2.0", - "licenseId": "MPL-2.0", + "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-only.json", + "referenceNumber": 439, + "name": "GNU Affero General Public License v3.0 only", + "licenseId": "AGPL-3.0-only", "seeAlso": [ - "https://www.mozilla.org/MPL/2.0/", - "https://opensource.org/licenses/MPL-2.0" + "https://www.gnu.org/licenses/agpl.txt", + "https://opensource.org/licenses/AGPL-3.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.0.1.html", + "reference": "https://spdx.org/licenses/AGPL-3.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.1.json", - "referenceNumber": 17, - "name": "Open LDAP Public License v2.0.1", - "licenseId": "OLDAP-2.0.1", + "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-or-later.json", + "referenceNumber": 254, + "name": "GNU Affero General Public License v3.0 or later", + "licenseId": "AGPL-3.0-or-later", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db6d68acd14e51ca3aab4428bf26522aa74873f0e" + "https://www.gnu.org/licenses/agpl.txt", + "https://opensource.org/licenses/AGPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Giftware.html", + "reference": "https://spdx.org/licenses/Aladdin.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Giftware.json", - "referenceNumber": 18, - "name": "Giftware License", - "licenseId": "Giftware", - "seeAlso": [ - "http://liballeg.org/license.html#allegro-4-the-giftware-license" - ], - "isOsiApproved": false - }, - { - "reference": "https://spdx.org/licenses/AGPL-1.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/AGPL-1.0.json", - "referenceNumber": 19, - "name": "Affero General Public License v1.0", - "licenseId": "AGPL-1.0", + "detailsUrl": "https://spdx.org/licenses/Aladdin.json", + "referenceNumber": 294, + "name": "Aladdin Free Public License", + "licenseId": "Aladdin", "seeAlso": [ - "http://www.affero.org/oagpl.html" + "http://pages.cs.wisc.edu/~ghost/doc/AFPL/6.01/Public.htm" ], "isOsiApproved": false, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/GPL-1.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0+.json", - "referenceNumber": 20, - "name": "GNU General Public License v1.0 or later", - "licenseId": "GPL-1.0+", - "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" - ], - "isOsiApproved": false + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/Eurosym.html", + "reference": "https://spdx.org/licenses/AMDPLPA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Eurosym.json", - "referenceNumber": 21, - "name": "Eurosym License", - "licenseId": "Eurosym", + "detailsUrl": "https://spdx.org/licenses/AMDPLPA.json", + "referenceNumber": 463, + "name": "AMD\u0027s plpa_map.c License", + "licenseId": "AMDPLPA", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Eurosym" + "https://fedoraproject.org/wiki/Licensing/AMD_plpa_map_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EPICS.html", + "reference": "https://spdx.org/licenses/AML.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EPICS.json", - "referenceNumber": 22, - "name": "EPICS Open License", - "licenseId": "EPICS", + "detailsUrl": "https://spdx.org/licenses/AML.json", + "referenceNumber": 117, + "name": "Apple MIT License", + "licenseId": "AML", "seeAlso": [ - "https://epics.anl.gov/license/open.php" + "https://fedoraproject.org/wiki/Licensing/Apple_MIT_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.0.html", + "reference": "https://spdx.org/licenses/AMPAS.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.json", - "referenceNumber": 23, - "name": "Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B)", - "licenseId": "OLDAP-2.0", + "detailsUrl": "https://spdx.org/licenses/AMPAS.json", + "referenceNumber": 223, + "name": "Academy of Motion Picture Arts and Sciences BSD", + "licenseId": "AMPAS", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcbf50f4e1185a21abd4c0a54d3f4341fe28f36ea" + "https://fedoraproject.org/wiki/Licensing/BSD#AMPASBSD" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-Protection.html", + "reference": "https://spdx.org/licenses/ANTLR-PD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-Protection.json", - "referenceNumber": 24, - "name": "BSD Protection License", - "licenseId": "BSD-Protection", + "detailsUrl": "https://spdx.org/licenses/ANTLR-PD.json", + "referenceNumber": 206, + "name": "ANTLR Software Rights Notice", + "licenseId": "ANTLR-PD", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/BSD_Protection_License" + "http://www.antlr2.org/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SAX-PD.html", + "reference": "https://spdx.org/licenses/ANTLR-PD-fallback.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SAX-PD.json", - "referenceNumber": 25, - "name": "Sax Public Domain Notice", - "licenseId": "SAX-PD", + "detailsUrl": "https://spdx.org/licenses/ANTLR-PD-fallback.json", + "referenceNumber": 256, + "name": "ANTLR Software Rights Notice with license fallback", + "licenseId": "ANTLR-PD-fallback", "seeAlso": [ - "http://www.saxproject.org/copying.html" + "http://www.antlr2.org/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.html", + "reference": "https://spdx.org/licenses/Apache-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.json", - "referenceNumber": 26, - "name": "Creative Commons Attribution Non Commercial 3.0 Germany", - "licenseId": "CC-BY-NC-3.0-DE", + "detailsUrl": "https://spdx.org/licenses/Apache-1.0.json", + "referenceNumber": 491, + "name": "Apache License 1.0", + "licenseId": "Apache-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/3.0/de/legalcode" + "http://www.apache.org/licenses/LICENSE-1.0" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.html", + "reference": "https://spdx.org/licenses/Apache-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.json", - "referenceNumber": 27, - "name": "GNU Free Documentation License v1.3 only - no invariants", - "licenseId": "GFDL-1.3-no-invariants-only", + "detailsUrl": "https://spdx.org/licenses/Apache-1.1.json", + "referenceNumber": 153, + "name": "Apache License 1.1", + "licenseId": "Apache-1.1", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "http://apache.org/licenses/LICENSE-1.1", + "https://opensource.org/licenses/Apache-1.1" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/HPND.html", + "reference": "https://spdx.org/licenses/Apache-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/HPND.json", - "referenceNumber": 28, - "name": "Historical Permission Notice and Disclaimer", - "licenseId": "HPND", + "detailsUrl": "https://spdx.org/licenses/Apache-2.0.json", + "referenceNumber": 78, + "name": "Apache License 2.0", + "licenseId": "Apache-2.0", "seeAlso": [ - "https://opensource.org/licenses/HPND" + "https://www.apache.org/licenses/LICENSE-2.0", + "https://opensource.org/licenses/Apache-2.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.html", + "reference": "https://spdx.org/licenses/APAFML.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.json", - "referenceNumber": 29, - "name": "Creative Commons Attribution Share Alike 3.0 Germany", - "licenseId": "CC-BY-SA-3.0-DE", + "detailsUrl": "https://spdx.org/licenses/APAFML.json", + "referenceNumber": 273, + "name": "Adobe Postscript AFM License", + "licenseId": "APAFML", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/3.0/de/legalcode" + "https://fedoraproject.org/wiki/Licensing/AdobePostscriptAFM" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-3.0.html", + "reference": "https://spdx.org/licenses/APL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0.json", - "referenceNumber": 30, - "name": "Creative Commons Attribution Share Alike 3.0 Unported", - "licenseId": "CC-BY-SA-3.0", + "detailsUrl": "https://spdx.org/licenses/APL-1.0.json", + "referenceNumber": 316, + "name": "Adaptive Public License 1.0", + "licenseId": "APL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/3.0/legalcode" + "https://opensource.org/licenses/APL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.4.html", + "reference": "https://spdx.org/licenses/App-s2p.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.4.json", - "referenceNumber": 31, - "name": "Open LDAP Public License v2.4", - "licenseId": "OLDAP-2.4", + "detailsUrl": "https://spdx.org/licenses/App-s2p.json", + "referenceNumber": 448, + "name": "App::s2p License", + "licenseId": "App-s2p", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcd1284c4a91a8a380d904eee68d1583f989ed386" + "https://fedoraproject.org/wiki/Licensing/App-s2p" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Zimbra-1.4.html", + "reference": "https://spdx.org/licenses/APSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zimbra-1.4.json", - "referenceNumber": 32, - "name": "Zimbra Public License v1.4", - "licenseId": "Zimbra-1.4", + "detailsUrl": "https://spdx.org/licenses/APSL-1.0.json", + "referenceNumber": 66, + "name": "Apple Public Source License 1.0", + "licenseId": "APSL-1.0", "seeAlso": [ - "http://www.zimbra.com/legal/zimbra-public-license-1-4" + "https://fedoraproject.org/wiki/Licensing/Apple_Public_Source_License_1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/PostgreSQL.html", + "reference": "https://spdx.org/licenses/APSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PostgreSQL.json", - "referenceNumber": 33, - "name": "PostgreSQL License", - "licenseId": "PostgreSQL", + "detailsUrl": "https://spdx.org/licenses/APSL-1.1.json", + "referenceNumber": 277, + "name": "Apple Public Source License 1.1", + "licenseId": "APSL-1.1", "seeAlso": [ - "http://www.postgresql.org/about/licence", - "https://opensource.org/licenses/PostgreSQL" + "http://www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Noweb.html", + "reference": "https://spdx.org/licenses/APSL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Noweb.json", - "referenceNumber": 34, - "name": "Noweb License", - "licenseId": "Noweb", + "detailsUrl": "https://spdx.org/licenses/APSL-1.2.json", + "referenceNumber": 278, + "name": "Apple Public Source License 1.2", + "licenseId": "APSL-1.2", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Noweb" + "http://www.samurajdata.se/opensource/mirror/licenses/apsl.php" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/NIST-PD-fallback.html", + "reference": "https://spdx.org/licenses/APSL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NIST-PD-fallback.json", - "referenceNumber": 35, - "name": "NIST Public Domain Notice with license fallback", - "licenseId": "NIST-PD-fallback", + "detailsUrl": "https://spdx.org/licenses/APSL-2.0.json", + "referenceNumber": 65, + "name": "Apple Public Source License 2.0", + "licenseId": "APSL-2.0", "seeAlso": [ - "https://github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE", - "https://github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst" + "http://www.opensource.apple.com/license/apsl/" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Mup.html", + "reference": "https://spdx.org/licenses/Arphic-1999.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Mup.json", - "referenceNumber": 36, - "name": "Mup License", - "licenseId": "Mup", - "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Mup" + "detailsUrl": "https://spdx.org/licenses/Arphic-1999.json", + "referenceNumber": 168, + "name": "Arphic Public License", + "licenseId": "Arphic-1999", + "seeAlso": [ + "http://ftp.gnu.org/gnu/non-gnu/chinese-fonts-truetype/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0.json", - "referenceNumber": 37, - "name": "GNU General Public License v3.0 only", - "licenseId": "GPL-3.0", + "reference": "https://spdx.org/licenses/Artistic-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Artistic-1.0.json", + "referenceNumber": 70, + "name": "Artistic License 1.0", + "licenseId": "Artistic-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "https://opensource.org/licenses/Artistic-1.0" ], "isOsiApproved": true, - "isFsfLibre": true + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/checkmk.html", + "reference": "https://spdx.org/licenses/Artistic-1.0-cl8.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/checkmk.json", - "referenceNumber": 38, - "name": "Checkmk License", - "licenseId": "checkmk", + "detailsUrl": "https://spdx.org/licenses/Artistic-1.0-cl8.json", + "referenceNumber": 264, + "name": "Artistic License 1.0 w/clause 8", + "licenseId": "Artistic-1.0-cl8", "seeAlso": [ - "https://github.com/libcheck/check/blob/master/checkmk/checkmk.in" + "https://opensource.org/licenses/Artistic-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Parity-6.0.0.html", + "reference": "https://spdx.org/licenses/Artistic-1.0-Perl.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Parity-6.0.0.json", - "referenceNumber": 39, - "name": "The Parity Public License 6.0.0", - "licenseId": "Parity-6.0.0", + "detailsUrl": "https://spdx.org/licenses/Artistic-1.0-Perl.json", + "referenceNumber": 481, + "name": "Artistic License 1.0 (Perl)", + "licenseId": "Artistic-1.0-Perl", "seeAlso": [ - "https://paritylicense.com/versions/6.0.0.html" + "http://dev.perl.org/licenses/artistic.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Linux-OpenIB.html", + "reference": "https://spdx.org/licenses/Artistic-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Linux-OpenIB.json", - "referenceNumber": 40, - "name": "Linux Kernel Variant of OpenIB.org license", - "licenseId": "Linux-OpenIB", + "detailsUrl": "https://spdx.org/licenses/Artistic-2.0.json", + "referenceNumber": 432, + "name": "Artistic License 2.0", + "licenseId": "Artistic-2.0", "seeAlso": [ - "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h" + "http://www.perlfoundation.org/artistic_license_2_0", + "https://www.perlfoundation.org/artistic-license-20.html", + "https://opensource.org/licenses/artistic-license-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/TCL.html", + "reference": "https://spdx.org/licenses/Baekmuk.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TCL.json", - "referenceNumber": 41, - "name": "TCL/TK License", - "licenseId": "TCL", + "detailsUrl": "https://spdx.org/licenses/Baekmuk.json", + "referenceNumber": 28, + "name": "Baekmuk License", + "licenseId": "Baekmuk", "seeAlso": [ - "http://www.tcl.tk/software/tcltk/license.html", - "https://fedoraproject.org/wiki/Licensing/TCL" + "https://fedoraproject.org/wiki/Licensing:Baekmuk?rd\u003dLicensing/Baekmuk" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CERN-OHL-P-2.0.html", + "reference": "https://spdx.org/licenses/Bahyph.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-P-2.0.json", - "referenceNumber": 42, - "name": "CERN Open Hardware Licence Version 2 - Permissive", - "licenseId": "CERN-OHL-P-2.0", + "detailsUrl": "https://spdx.org/licenses/Bahyph.json", + "referenceNumber": 220, + "name": "Bahyph License", + "licenseId": "Bahyph", "seeAlso": [ - "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + "https://fedoraproject.org/wiki/Licensing/Bahyph" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EPL-1.0.html", + "reference": "https://spdx.org/licenses/Barr.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EPL-1.0.json", - "referenceNumber": 43, - "name": "Eclipse Public License 1.0", - "licenseId": "EPL-1.0", + "detailsUrl": "https://spdx.org/licenses/Barr.json", + "referenceNumber": 442, + "name": "Barr License", + "licenseId": "Barr", "seeAlso": [ - "http://www.eclipse.org/legal/epl-v10.html", - "https://opensource.org/licenses/EPL-1.0" + "https://fedoraproject.org/wiki/Licensing/Barr" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EUPL-1.2.html", + "reference": "https://spdx.org/licenses/Beerware.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUPL-1.2.json", - "referenceNumber": 44, - "name": "European Union Public License 1.2", - "licenseId": "EUPL-1.2", + "detailsUrl": "https://spdx.org/licenses/Beerware.json", + "referenceNumber": 126, + "name": "Beerware License", + "licenseId": "Beerware", "seeAlso": [ - "https://joinup.ec.europa.eu/page/eupl-text-11-12", - "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl_v1.2_en.pdf", - "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/2020-03/EUPL-1.2%20EN.txt", - "https://joinup.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt", - "http://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri\u003dCELEX:32017D0863", - "https://opensource.org/licenses/EUPL-1.2" + "https://fedoraproject.org/wiki/Licensing/Beerware", + "https://people.freebsd.org/~phk/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/D-FSL-1.0.html", + "reference": "https://spdx.org/licenses/Bitstream-Charter.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/D-FSL-1.0.json", - "referenceNumber": 45, - "name": "Deutsche Freie Software Lizenz", - "licenseId": "D-FSL-1.0", + "detailsUrl": "https://spdx.org/licenses/Bitstream-Charter.json", + "referenceNumber": 446, + "name": "Bitstream Charter Font License", + "licenseId": "Bitstream-Charter", "seeAlso": [ - "http://www.dipp.nrw.de/d-fsl/lizenzen/", - "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/de/D-FSL-1_0_de.txt", - "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/en/D-FSL-1_0_en.txt", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/deutsche-freie-software-lizenz", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/german-free-software-license", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_de.txt/at_download/file", - "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_en.txt/at_download/file" + "https://fedoraproject.org/wiki/Licensing/Charter#License_Text", + "https://raw.githubusercontent.com/blackhole89/notekit/master/data/fonts/Charter%20license.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Rdisc.html", + "reference": "https://spdx.org/licenses/Bitstream-Vera.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Rdisc.json", - "referenceNumber": 46, - "name": "Rdisc License", - "licenseId": "Rdisc", + "detailsUrl": "https://spdx.org/licenses/Bitstream-Vera.json", + "referenceNumber": 58, + "name": "Bitstream Vera Font License", + "licenseId": "Bitstream-Vera", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Rdisc_License" + "https://web.archive.org/web/20080207013128/http://www.gnome.org/fonts/", + "https://docubrain.com/sites/default/files/licenses/bitstream-vera.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.6.html", + "reference": "https://spdx.org/licenses/BitTorrent-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.6.json", - "referenceNumber": 47, - "name": "Open LDAP Public License v2.6", - "licenseId": "OLDAP-2.6", + "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.0.json", + "referenceNumber": 203, + "name": "BitTorrent Open Source License v1.0", + "licenseId": "BitTorrent-1.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d1cae062821881f41b73012ba816434897abf4205" + "http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1\u003d1.1\u0026r2\u003d1.1.1.1\u0026diff_format\u003ds" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Artistic-1.0-cl8.html", + "reference": "https://spdx.org/licenses/BitTorrent-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Artistic-1.0-cl8.json", - "referenceNumber": 48, - "name": "Artistic License 1.0 w/clause 8", - "licenseId": "Artistic-1.0-cl8", + "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.1.json", + "referenceNumber": 412, + "name": "BitTorrent Open Source License v1.1", + "licenseId": "BitTorrent-1.1", "seeAlso": [ - "https://opensource.org/licenses/Artistic-1.0" + "http://directory.fsf.org/wiki/License:BitTorrentOSL1.1" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SNIA.html", + "reference": "https://spdx.org/licenses/blessing.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SNIA.json", - "referenceNumber": 49, - "name": "SNIA Public License 1.1", - "licenseId": "SNIA", + "detailsUrl": "https://spdx.org/licenses/blessing.json", + "referenceNumber": 105, + "name": "SQLite Blessing", + "licenseId": "blessing", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/SNIA_Public_License" + "https://www.sqlite.org/src/artifact/e33a4df7e32d742a?ln\u003d4-9", + "https://sqlite.org/src/artifact/df5091916dbb40e6" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.html", + "reference": "https://spdx.org/licenses/BlueOak-1.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.json", - "referenceNumber": 50, - "name": "Creative Commons Attribution Non Commercial Share Alike 4.0 International", - "licenseId": "CC-BY-NC-SA-4.0", + "detailsUrl": "https://spdx.org/licenses/BlueOak-1.0.0.json", + "referenceNumber": 173, + "name": "Blue Oak Model License 1.0.0", + "licenseId": "BlueOak-1.0.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode" + "https://blueoakcouncil.org/license/1.0.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ICU.html", + "reference": "https://spdx.org/licenses/Borceux.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ICU.json", - "referenceNumber": 51, - "name": "ICU License", - "licenseId": "ICU", + "detailsUrl": "https://spdx.org/licenses/Borceux.json", + "referenceNumber": 29, + "name": "Borceux license", + "licenseId": "Borceux", "seeAlso": [ - "http://source.icu-project.org/repos/icu/icu/trunk/license.html" + "https://fedoraproject.org/wiki/Licensing/Borceux" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FSFAP.html", + "reference": "https://spdx.org/licenses/BSD-1-Clause.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FSFAP.json", - "referenceNumber": 52, - "name": "FSF All Permissive License", - "licenseId": "FSFAP", + "detailsUrl": "https://spdx.org/licenses/BSD-1-Clause.json", + "referenceNumber": 42, + "name": "BSD 1-Clause License", + "licenseId": "BSD-1-Clause", "seeAlso": [ - "https://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html" + "https://svnweb.freebsd.org/base/head/include/ifaddrs.h?revision\u003d326823" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/MIT-Modern-Variant.html", + "reference": "https://spdx.org/licenses/BSD-2-Clause.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-Modern-Variant.json", - "referenceNumber": 53, - "name": "MIT License Modern Variant", - "licenseId": "MIT-Modern-Variant", + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause.json", + "referenceNumber": 208, + "name": "BSD 2-Clause \"Simplified\" License", + "licenseId": "BSD-2-Clause", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:MIT#Modern_Variants", - "https://ptolemy.berkeley.edu/copyright.htm", - "https://pirlwww.lpl.arizona.edu/resources/guide/software/PerlTk/Tixlic.html" + "https://opensource.org/licenses/BSD-2-Clause" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/JSON.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/JSON.json", - "referenceNumber": 54, - "name": "JSON License", - "licenseId": "JSON", + "reference": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.json", + "referenceNumber": 129, + "name": "BSD 2-Clause FreeBSD License", + "licenseId": "BSD-2-Clause-FreeBSD", "seeAlso": [ - "http://www.json.org/license.html" + "http://www.freebsd.org/copyright/freebsd-license.html" ], "isOsiApproved": false, - "isFsfLibre": false + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LZMA-SDK-9.11-to-9.20.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LZMA-SDK-9.11-to-9.20.json", - "referenceNumber": 55, - "name": "LZMA SDK License (versions 9.11 to 9.20)", - "licenseId": "LZMA-SDK-9.11-to-9.20", + "reference": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.json", + "referenceNumber": 196, + "name": "BSD 2-Clause NetBSD License", + "licenseId": "BSD-2-Clause-NetBSD", "seeAlso": [ - "https://www.7-zip.org/sdk.html", - "https://sourceforge.net/projects/sevenzip/files/LZMA%20SDK/" + "http://www.netbsd.org/about/redistribution.html#default" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/0BSD.html", + "reference": "https://spdx.org/licenses/BSD-2-Clause-Patent.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/0BSD.json", - "referenceNumber": 56, - "name": "BSD Zero Clause License", - "licenseId": "0BSD", + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Patent.json", + "referenceNumber": 7, + "name": "BSD-2-Clause Plus Patent License", + "licenseId": "BSD-2-Clause-Patent", "seeAlso": [ - "http://landley.net/toybox/license.html", - "https://opensource.org/licenses/0BSD" + "https://opensource.org/licenses/BSDplusPatent" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/LPPL-1.0.html", + "reference": "https://spdx.org/licenses/BSD-2-Clause-Views.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.0.json", - "referenceNumber": 57, - "name": "LaTeX Project Public License v1.0", - "licenseId": "LPPL-1.0", + "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Views.json", + "referenceNumber": 502, + "name": "BSD 2-Clause with views sentence", + "licenseId": "BSD-2-Clause-Views", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-0.txt" + "http://www.freebsd.org/copyright/freebsd-license.html", + "https://people.freebsd.org/~ivoras/wine/patch-wine-nvidia.sh", + "https://github.com/protegeproject/protege/blob/master/license.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/mpich2.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/mpich2.json", - "referenceNumber": 58, - "name": "mpich2 License", - "licenseId": "mpich2", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause.json", + "referenceNumber": 39, + "name": "BSD 3-Clause \"New\" or \"Revised\" License", + "licenseId": "BSD-3-Clause", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT" + "https://opensource.org/licenses/BSD-3-Clause", + "https://www.eclipse.org/org/documents/edl-v10.php" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/TAPR-OHL-1.0.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Attribution.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TAPR-OHL-1.0.json", - "referenceNumber": 59, - "name": "TAPR Open Hardware License v1.0", - "licenseId": "TAPR-OHL-1.0", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Attribution.json", + "referenceNumber": 353, + "name": "BSD with attribution", + "licenseId": "BSD-3-Clause-Attribution", "seeAlso": [ - "https://www.tapr.org/OHL" + "https://fedoraproject.org/wiki/Licensing/BSD_with_Attribution" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/blessing.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Clear.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/blessing.json", - "referenceNumber": 60, - "name": "SQLite Blessing", - "licenseId": "blessing", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Clear.json", + "referenceNumber": 397, + "name": "BSD 3-Clause Clear License", + "licenseId": "BSD-3-Clause-Clear", "seeAlso": [ - "https://www.sqlite.org/src/artifact/e33a4df7e32d742a?ln\u003d4-9", - "https://sqlite.org/src/artifact/df5091916dbb40e6" + "http://labs.metacarta.com/license-explanation.html#license" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Aladdin.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-LBNL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Aladdin.json", - "referenceNumber": 61, - "name": "Aladdin Free Public License", - "licenseId": "Aladdin", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-LBNL.json", + "referenceNumber": 301, + "name": "Lawrence Berkeley National Labs BSD variant license", + "licenseId": "BSD-3-Clause-LBNL", "seeAlso": [ - "http://pages.cs.wisc.edu/~ghost/doc/AFPL/6.01/Public.htm" + "https://fedoraproject.org/wiki/Licensing/LBNLBSD" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/MIT-advertising.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Modification.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-advertising.json", - "referenceNumber": 62, - "name": "Enlightenment License (e16)", - "licenseId": "MIT-advertising", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Modification.json", + "referenceNumber": 214, + "name": "BSD 3-Clause Modification", + "licenseId": "BSD-3-Clause-Modification", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT_With_Advertising" + "https://fedoraproject.org/wiki/Licensing:BSD#Modification_Variant" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LAL-1.3.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LAL-1.3.json", - "referenceNumber": 63, - "name": "Licence Art Libre 1.3", - "licenseId": "LAL-1.3", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.json", + "referenceNumber": 23, + "name": "BSD 3-Clause No Military License", + "licenseId": "BSD-3-Clause-No-Military-License", "seeAlso": [ - "https://artlibre.org/" + "https://gitlab.syncad.com/hive/dhive/-/blob/master/LICENSE", + "https://github.com/greymass/swift-eosio/blob/master/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/X11.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/X11.json", - "referenceNumber": 64, - "name": "X11 License", - "licenseId": "X11", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.json", + "referenceNumber": 393, + "name": "BSD 3-Clause No Nuclear License", + "licenseId": "BSD-3-Clause-No-Nuclear-License", "seeAlso": [ - "http://www.xfree86.org/3.3.6/COPYRIGHT2.html#3" + "http://download.oracle.com/otn-pub/java/licenses/bsd.txt?AuthParam\u003d1467140197_43d516ce1776bd08a58235a7785be1cc" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NOSL.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NOSL.json", - "referenceNumber": 65, - "name": "Netizen Open Source License", - "licenseId": "NOSL", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.json", + "referenceNumber": 356, + "name": "BSD 3-Clause No Nuclear License 2014", + "licenseId": "BSD-3-Clause-No-Nuclear-License-2014", "seeAlso": [ - "http://bits.netizen.com.au/licenses/NOSL/nosl.txt" + "https://java.net/projects/javaeetutorial/pages/BerkeleyLicense" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-Patent.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Patent.json", - "referenceNumber": 66, - "name": "BSD-2-Clause Plus Patent License", - "licenseId": "BSD-2-Clause-Patent", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.json", + "referenceNumber": 225, + "name": "BSD 3-Clause No Nuclear Warranty", + "licenseId": "BSD-3-Clause-No-Nuclear-Warranty", "seeAlso": [ - "https://opensource.org/licenses/BSDplusPatent" + "https://jogamp.org/git/?p\u003dgluegen.git;a\u003dblob_plain;f\u003dLICENSE.txt" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MulanPSL-1.0.html", + "reference": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MulanPSL-1.0.json", - "referenceNumber": 67, - "name": "Mulan Permissive Software License, Version 1", - "licenseId": "MulanPSL-1.0", + "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.json", + "referenceNumber": 389, + "name": "BSD 3-Clause Open MPI variant", + "licenseId": "BSD-3-Clause-Open-MPI", "seeAlso": [ - "https://license.coscl.org.cn/MulanPSL/", - "https://github.com/yuwenlong/longphp/blob/25dfb70cc2a466dc4bb55ba30901cbce08d164b5/LICENSE" + "https://www.open-mpi.org/community/license.php", + "http://www.netlib.org/lapack/LICENSE.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Spencer-86.html", + "reference": "https://spdx.org/licenses/BSD-4-Clause.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Spencer-86.json", - "referenceNumber": 68, - "name": "Spencer License 86", - "licenseId": "Spencer-86", + "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause.json", + "referenceNumber": 507, + "name": "BSD 4-Clause \"Original\" or \"Old\" License", + "licenseId": "BSD-4-Clause", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" + "http://directory.fsf.org/wiki/License:BSD_4Clause" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Net-SNMP.html", + "reference": "https://spdx.org/licenses/BSD-4-Clause-Shortened.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Net-SNMP.json", - "referenceNumber": 69, - "name": "Net-SNMP License", - "licenseId": "Net-SNMP", + "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-Shortened.json", + "referenceNumber": 8, + "name": "BSD 4 Clause Shortened", + "licenseId": "BSD-4-Clause-Shortened", "seeAlso": [ - "http://net-snmp.sourceforge.net/about/license.html" + "https://metadata.ftp-master.debian.org/changelogs//main/a/arpwatch/arpwatch_2.1a15-7_copyright" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-3.0-only.html", + "reference": "https://spdx.org/licenses/BSD-4-Clause-UC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-only.json", - "referenceNumber": 70, - "name": "GNU Lesser General Public License v3.0 only", - "licenseId": "LGPL-3.0-only", + "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-UC.json", + "referenceNumber": 259, + "name": "BSD-4-Clause (University of California-Specific)", + "licenseId": "BSD-4-Clause-UC", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", - "https://opensource.org/licenses/LGPL-3.0" + "http://www.freebsd.org/copyright/license.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AGPL-1.0-or-later.html", + "reference": "https://spdx.org/licenses/BSD-Protection.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-or-later.json", - "referenceNumber": 71, - "name": "Affero General Public License v1.0 or later", - "licenseId": "AGPL-1.0-or-later", + "detailsUrl": "https://spdx.org/licenses/BSD-Protection.json", + "referenceNumber": 241, + "name": "BSD Protection License", + "licenseId": "BSD-Protection", "seeAlso": [ - "http://www.affero.org/oagpl.html" + "https://fedoraproject.org/wiki/Licensing/BSD_Protection_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Apache-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Apache-1.0.json", - "referenceNumber": 72, - "name": "Apache License 1.0", - "licenseId": "Apache-1.0", - "seeAlso": [ - "http://www.apache.org/licenses/LICENSE-1.0" - ], - "isOsiApproved": false, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/Cube.html", + "reference": "https://spdx.org/licenses/BSD-Source-Code.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Cube.json", - "referenceNumber": 73, - "name": "Cube License", - "licenseId": "Cube", + "detailsUrl": "https://spdx.org/licenses/BSD-Source-Code.json", + "referenceNumber": 101, + "name": "BSD Source Code Attribution", + "licenseId": "BSD-Source-Code", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Cube" + "https://github.com/robbiehanson/CocoaHTTPServer/blob/master/LICENSE.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-3.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0+.json", - "referenceNumber": 74, - "name": "GNU Lesser General Public License v3.0 or later", - "licenseId": "LGPL-3.0+", + "reference": "https://spdx.org/licenses/BSL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/BSL-1.0.json", + "referenceNumber": 140, + "name": "Boost Software License 1.0", + "licenseId": "BSL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", - "https://opensource.org/licenses/LGPL-3.0" + "http://www.boost.org/LICENSE_1_0.txt", + "https://opensource.org/licenses/BSL-1.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NCGL-UK-2.0.html", + "reference": "https://spdx.org/licenses/BUSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NCGL-UK-2.0.json", - "referenceNumber": 75, - "name": "Non-Commercial Government Licence", - "licenseId": "NCGL-UK-2.0", + "detailsUrl": "https://spdx.org/licenses/BUSL-1.1.json", + "referenceNumber": 250, + "name": "Business Source License 1.1", + "licenseId": "BUSL-1.1", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/non-commercial-government-licence/version/2/" + "https://mariadb.com/bsl11/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-AT.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-AT.json", - "referenceNumber": 76, - "name": "Creative Commons Attribution 3.0 Austria", - "licenseId": "CC-BY-3.0-AT", + "reference": "https://spdx.org/licenses/bzip2-1.0.5.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.5.json", + "referenceNumber": 30, + "name": "bzip2 and libbzip2 License v1.0.5", + "licenseId": "bzip2-1.0.5", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/at/legalcode" + "https://sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html", + "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGL-UK-2.0.html", + "reference": "https://spdx.org/licenses/bzip2-1.0.6.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-UK-2.0.json", - "referenceNumber": 77, - "name": "Open Government Licence v2.0", - "licenseId": "OGL-UK-2.0", + "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.6.json", + "referenceNumber": 306, + "name": "bzip2 and libbzip2 License v1.0.6", + "licenseId": "bzip2-1.0.6", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" + "https://sourceware.org/git/?p\u003dbzip2.git;a\u003dblob;f\u003dLICENSE;hb\u003dbzip2-1.0.6", + "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-1.0.html", + "reference": "https://spdx.org/licenses/C-UDA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-1.0.json", - "referenceNumber": 78, - "name": "Creative Commons Attribution 1.0 Generic", - "licenseId": "CC-BY-1.0", + "detailsUrl": "https://spdx.org/licenses/C-UDA-1.0.json", + "referenceNumber": 279, + "name": "Computational Use of Data Agreement v1.0", + "licenseId": "C-UDA-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by/1.0/legalcode" + "https://github.com/microsoft/Computational-Use-of-Data-Agreement/blob/master/C-UDA-1.0.md", + "https://cdla.dev/computational-use-of-data-agreement-v1-0/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MakeIndex.html", + "reference": "https://spdx.org/licenses/CAL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MakeIndex.json", - "referenceNumber": 79, - "name": "MakeIndex License", - "licenseId": "MakeIndex", + "detailsUrl": "https://spdx.org/licenses/CAL-1.0.json", + "referenceNumber": 504, + "name": "Cryptographic Autonomy License 1.0", + "licenseId": "CAL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MakeIndex" + "http://cryptographicautonomylicense.com/license-text.html", + "https://opensource.org/licenses/CAL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/LAL-1.2.html", + "reference": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LAL-1.2.json", - "referenceNumber": 80, - "name": "Licence Art Libre 1.2", - "licenseId": "LAL-1.2", + "detailsUrl": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.json", + "referenceNumber": 329, + "name": "Cryptographic Autonomy License 1.0 (Combined Work Exception)", + "licenseId": "CAL-1.0-Combined-Work-Exception", "seeAlso": [ - "http://artlibre.org/licence/lal/licence-art-libre-12/" + "http://cryptographicautonomylicense.com/license-text.html", + "https://opensource.org/licenses/CAL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.html", + "reference": "https://spdx.org/licenses/Caldera.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.json", - "referenceNumber": 81, - "name": "GNU Free Documentation License v1.2 or later - no invariants", - "licenseId": "GFDL-1.2-no-invariants-or-later", + "detailsUrl": "https://spdx.org/licenses/Caldera.json", + "referenceNumber": 333, + "name": "Caldera License", + "licenseId": "Caldera", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-or-later.html", + "reference": "https://spdx.org/licenses/CATOSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-or-later.json", - "referenceNumber": 82, - "name": "GNU Free Documentation License v1.2 or later", - "licenseId": "GFDL-1.2-or-later", + "detailsUrl": "https://spdx.org/licenses/CATOSL-1.1.json", + "referenceNumber": 451, + "name": "Computer Associates Trusted Open Source License 1.1", + "licenseId": "CATOSL-1.1", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://opensource.org/licenses/CATOSL-1.1" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GLWTPL.html", + "reference": "https://spdx.org/licenses/CC-BY-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GLWTPL.json", - "referenceNumber": 83, - "name": "Good Luck With That Public License", - "licenseId": "GLWTPL", + "detailsUrl": "https://spdx.org/licenses/CC-BY-1.0.json", + "referenceNumber": 163, + "name": "Creative Commons Attribution 1.0 Generic", + "licenseId": "CC-BY-1.0", "seeAlso": [ - "https://github.com/me-shaon/GLWTPL/commit/da5f6bc734095efbacb442c0b31e33a65b9d6e85" + "https://creativecommons.org/licenses/by/1.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OPL-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OPL-1.0.json", - "referenceNumber": 84, - "name": "Open Public License v1.0", - "licenseId": "OPL-1.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.0.json", + "referenceNumber": 437, + "name": "Creative Commons Attribution 2.0 Generic", + "licenseId": "CC-BY-2.0", "seeAlso": [ - "http://old.koalateam.com/jackaroo/OPL_1_0.TXT", - "https://fedoraproject.org/wiki/Licensing/Open_Public_License" + "https://creativecommons.org/licenses/by/2.0/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Unicode-DFS-2016.html", + "reference": "https://spdx.org/licenses/CC-BY-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2016.json", - "referenceNumber": 85, - "name": "Unicode License Agreement - Data Files and Software (2016)", - "licenseId": "Unicode-DFS-2016", + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5.json", + "referenceNumber": 55, + "name": "Creative Commons Attribution 2.5 Generic", + "licenseId": "CC-BY-2.5", "seeAlso": [ - "http://www.unicode.org/copyright.html" + "https://creativecommons.org/licenses/by/2.5/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Parity-7.0.0.html", + "reference": "https://spdx.org/licenses/CC-BY-2.5-AU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Parity-7.0.0.json", - "referenceNumber": 86, - "name": "The Parity Public License 7.0.0", - "licenseId": "Parity-7.0.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5-AU.json", + "referenceNumber": 34, + "name": "Creative Commons Attribution 2.5 Australia", + "licenseId": "CC-BY-2.5-AU", "seeAlso": [ - "https://paritylicense.com/versions/7.0.0.html" + "https://creativecommons.org/licenses/by/2.5/au/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-LBNL.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-LBNL.json", - "referenceNumber": 87, - "name": "Lawrence Berkeley National Labs BSD variant license", - "licenseId": "BSD-3-Clause-LBNL", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0.json", + "referenceNumber": 490, + "name": "Creative Commons Attribution 3.0 Unported", + "licenseId": "CC-BY-3.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/LBNLBSD" + "https://creativecommons.org/licenses/by/3.0/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Community-Spec-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-AT.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Community-Spec-1.0.json", - "referenceNumber": 88, - "name": "Community Specification License 1.0", - "licenseId": "Community-Spec-1.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-AT.json", + "referenceNumber": 80, + "name": "Creative Commons Attribution 3.0 Austria", + "licenseId": "CC-BY-3.0-AT", "seeAlso": [ - "https://github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md" + "https://creativecommons.org/licenses/by/3.0/at/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.json", - "referenceNumber": 89, - "name": "GNU Free Documentation License v1.1 only - no invariants", - "licenseId": "GFDL-1.1-no-invariants-only", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-DE.json", + "referenceNumber": 166, + "name": "Creative Commons Attribution 3.0 Germany", + "licenseId": "CC-BY-3.0-DE", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://creativecommons.org/licenses/by/3.0/de/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AGPL-1.0-only.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-IGO.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-1.0-only.json", - "referenceNumber": 90, - "name": "Affero General Public License v1.0 only", - "licenseId": "AGPL-1.0-only", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-IGO.json", + "referenceNumber": 419, + "name": "Creative Commons Attribution 3.0 IGO", + "licenseId": "CC-BY-3.0-IGO", "seeAlso": [ - "http://www.affero.org/oagpl.html" + "https://creativecommons.org/licenses/by/3.0/igo/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/RSCPL.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-NL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RSCPL.json", - "referenceNumber": 91, - "name": "Ricoh Source Code Public License", - "licenseId": "RSCPL", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-NL.json", + "referenceNumber": 455, + "name": "Creative Commons Attribution 3.0 Netherlands", + "licenseId": "CC-BY-3.0-NL", "seeAlso": [ - "http://wayback.archive.org/web/20060715140826/http://www.risource.org/RPL/RPL-1.0A.shtml", - "https://opensource.org/licenses/RSCPL" + "https://creativecommons.org/licenses/by/3.0/nl/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SGI-B-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-3.0-US.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SGI-B-1.1.json", - "referenceNumber": 92, - "name": "SGI Free Software License B v1.1", - "licenseId": "SGI-B-1.1", + "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-US.json", + "referenceNumber": 492, + "name": "Creative Commons Attribution 3.0 United States", + "licenseId": "CC-BY-3.0-US", "seeAlso": [ - "http://oss.sgi.com/projects/FreeB/" + "https://creativecommons.org/licenses/by/3.0/us/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MIT-enna.html", + "reference": "https://spdx.org/licenses/CC-BY-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-enna.json", - "referenceNumber": 93, - "name": "enna License", - "licenseId": "MIT-enna", + "detailsUrl": "https://spdx.org/licenses/CC-BY-4.0.json", + "referenceNumber": 433, + "name": "Creative Commons Attribution 4.0 International", + "licenseId": "CC-BY-4.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT#enna" + "https://creativecommons.org/licenses/by/4.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Sleepycat.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Sleepycat.json", - "referenceNumber": 94, - "name": "Sleepycat License", - "licenseId": "Sleepycat", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-1.0.json", + "referenceNumber": 212, + "name": "Creative Commons Attribution Non Commercial 1.0 Generic", + "licenseId": "CC-BY-NC-1.0", "seeAlso": [ - "https://opensource.org/licenses/Sleepycat" + "https://creativecommons.org/licenses/by-nc/1.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/gSOAP-1.3b.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/gSOAP-1.3b.json", - "referenceNumber": 95, - "name": "gSOAP Public License v1.3b", - "licenseId": "gSOAP-1.3b", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.0.json", + "referenceNumber": 291, + "name": "Creative Commons Attribution Non Commercial 2.0 Generic", + "licenseId": "CC-BY-NC-2.0", "seeAlso": [ - "http://www.cs.fsu.edu/~engelen/license.html" + "https://creativecommons.org/licenses/by-nc/2.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CDLA-Sharing-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDLA-Sharing-1.0.json", - "referenceNumber": 96, - "name": "Community Data License Agreement Sharing 1.0", - "licenseId": "CDLA-Sharing-1.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.5.json", + "referenceNumber": 334, + "name": "Creative Commons Attribution Non Commercial 2.5 Generic", + "licenseId": "CC-BY-NC-2.5", "seeAlso": [ - "https://cdla.io/sharing-1-0" + "https://creativecommons.org/licenses/by-nc/2.5/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/Intel.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Intel.json", - "referenceNumber": 97, - "name": "Intel Open Source License", - "licenseId": "Intel", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0.json", + "referenceNumber": 350, + "name": "Creative Commons Attribution Non Commercial 3.0 Unported", + "licenseId": "CC-BY-NC-3.0", "seeAlso": [ - "https://opensource.org/licenses/Intel" + "https://creativecommons.org/licenses/by-nc/3.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CECILL-C.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-C.json", - "referenceNumber": 98, - "name": "CeCILL-C Free Software License Agreement", - "licenseId": "CECILL-C", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0-DE.json", + "referenceNumber": 475, + "name": "Creative Commons Attribution Non Commercial 3.0 Germany", + "licenseId": "CC-BY-NC-3.0-DE", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html" + "https://creativecommons.org/licenses/by-nc/3.0/de/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ZPL-2.1.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ZPL-2.1.json", - "referenceNumber": 99, - "name": "Zope Public License 2.1", - "licenseId": "ZPL-2.1", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-4.0.json", + "referenceNumber": 90, + "name": "Creative Commons Attribution Non Commercial 4.0 International", + "licenseId": "CC-BY-NC-4.0", "seeAlso": [ - "http://old.zope.org/Resources/ZPL/" + "https://creativecommons.org/licenses/by-nc/4.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/MIT-open-group.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-open-group.json", - "referenceNumber": 100, - "name": "MIT Open Group variant", - "licenseId": "MIT-open-group", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.json", + "referenceNumber": 5, + "name": "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + "licenseId": "CC-BY-NC-ND-1.0", "seeAlso": [ - "https://gitlab.freedesktop.org/xorg/app/iceauth/-/blob/master/COPYING", - "https://gitlab.freedesktop.org/xorg/app/xvinfo/-/blob/master/COPYING", - "https://gitlab.freedesktop.org/xorg/app/xsetroot/-/blob/master/COPYING", - "https://gitlab.freedesktop.org/xorg/app/xauth/-/blob/master/COPYING" + "https://creativecommons.org/licenses/by-nd-nc/1.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Zimbra-1.3.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zimbra-1.3.json", - "referenceNumber": 101, - "name": "Zimbra Public License v1.3", - "licenseId": "Zimbra-1.3", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.json", + "referenceNumber": 274, + "name": "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + "licenseId": "CC-BY-NC-ND-2.0", "seeAlso": [ - "http://web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html" + "https://creativecommons.org/licenses/by-nc-nd/2.0/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.json", - "referenceNumber": 102, - "name": "GNU General Public License v3.0 w/Autoconf exception", - "licenseId": "GPL-3.0-with-autoconf-exception", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.json", + "referenceNumber": 9, + "name": "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + "licenseId": "CC-BY-NC-ND-2.5", "seeAlso": [ - "https://www.gnu.org/licenses/autoconf-exception-3.0.html" + "https://creativecommons.org/licenses/by-nc-nd/2.5/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TU-Berlin-2.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TU-Berlin-2.0.json", - "referenceNumber": 103, - "name": "Technische Universitaet Berlin License 2.0", - "licenseId": "TU-Berlin-2.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.json", + "referenceNumber": 497, + "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + "licenseId": "CC-BY-NC-ND-3.0", "seeAlso": [ - "https://github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt" + "https://creativecommons.org/licenses/by-nc-nd/3.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/wxWindows.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/wxWindows.json", - "referenceNumber": 104, - "name": "wxWindows Library License", - "licenseId": "wxWindows", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json", + "referenceNumber": 399, + "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + "licenseId": "CC-BY-NC-ND-3.0-DE", "seeAlso": [ - "https://opensource.org/licenses/WXwindows" + "https://creativecommons.org/licenses/by-nc-nd/3.0/de/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0.json", - "referenceNumber": 105, - "name": "GNU Library General Public License v2 only", - "licenseId": "LGPL-2.0", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json", + "referenceNumber": 370, + "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + "licenseId": "CC-BY-NC-ND-3.0-IGO", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + "https://creativecommons.org/licenses/by-nc-nd/3.0/igo/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/PDDL-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PDDL-1.0.json", - "referenceNumber": 106, - "name": "Open Data Commons Public Domain Dedication \u0026 License 1.0", - "licenseId": "PDDL-1.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.json", + "referenceNumber": 310, + "name": "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + "licenseId": "CC-BY-NC-ND-4.0", "seeAlso": [ - "http://opendatacommons.org/licenses/pddl/1.0/", - "https://opendatacommons.org/licenses/pddl/" + "https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Frameworx-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Frameworx-1.0.json", - "referenceNumber": 107, - "name": "Frameworx Open License 1.0", - "licenseId": "Frameworx-1.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.json", + "referenceNumber": 369, + "name": "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + "licenseId": "CC-BY-NC-SA-1.0", "seeAlso": [ - "https://opensource.org/licenses/Frameworx-1.0" + "https://creativecommons.org/licenses/by-nc-sa/1.0/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-FreeBSD.json", - "referenceNumber": 108, - "name": "BSD 2-Clause FreeBSD License", - "licenseId": "BSD-2-Clause-FreeBSD", - "seeAlso": [ - "http://www.freebsd.org/copyright/freebsd-license.html" - ], - "isOsiApproved": false, - "isFsfLibre": true + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.json", + "referenceNumber": 180, + "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + "licenseId": "CC-BY-NC-SA-2.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nc-sa/2.0/legalcode" + ], + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AGPL-3.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/AGPL-3.0.json", - "referenceNumber": 109, - "name": "GNU Affero General Public License v3.0", - "licenseId": "AGPL-3.0", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE.json", + "referenceNumber": 132, + "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + "licenseId": "CC-BY-NC-SA-2.0-DE", "seeAlso": [ - "https://www.gnu.org/licenses/agpl.txt", - "https://opensource.org/licenses/AGPL-3.0" + "https://creativecommons.org/licenses/by-nc-sa/2.0/de/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/YPL-1.1.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/YPL-1.1.json", - "referenceNumber": 110, - "name": "Yahoo! Public License v1.1", - "licenseId": "YPL-1.1", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json", + "referenceNumber": 380, + "name": "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + "licenseId": "CC-BY-NC-SA-2.0-FR", "seeAlso": [ - "http://www.zimbra.com/license/yahoo_public_license_1.1.html" + "https://creativecommons.org/licenses/by-nc-sa/2.0/fr/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-2.5.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5.json", - "referenceNumber": 111, - "name": "Creative Commons Attribution 2.5 Generic", - "licenseId": "CC-BY-2.5", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json", + "referenceNumber": 296, + "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + "licenseId": "CC-BY-NC-SA-2.0-UK", "seeAlso": [ - "https://creativecommons.org/licenses/by/2.5/legalcode" + "https://creativecommons.org/licenses/by-nc-sa/2.0/uk/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-4.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-4.0.json", - "referenceNumber": 112, - "name": "Creative Commons Attribution Share Alike 4.0 International", - "licenseId": "CC-BY-SA-4.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.json", + "referenceNumber": 33, + "name": "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + "licenseId": "CC-BY-NC-SA-2.5", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/4.0/legalcode" + "https://creativecommons.org/licenses/by-nc-sa/2.5/legalcode" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Adobe-2006.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Adobe-2006.json", - "referenceNumber": 113, - "name": "Adobe Systems Incorporated Source Code License Agreement", - "licenseId": "Adobe-2006", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.json", + "referenceNumber": 337, + "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + "licenseId": "CC-BY-NC-SA-3.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AdobeLicense" + "https://creativecommons.org/licenses/by-nc-sa/3.0/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.1+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.1+.json", - "referenceNumber": 114, - "name": "GNU Library General Public License v2.1 or later", - "licenseId": "LGPL-2.1+", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json", + "referenceNumber": 269, + "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + "licenseId": "CC-BY-NC-SA-3.0-DE", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", - "https://opensource.org/licenses/LGPL-2.1" + "https://creativecommons.org/licenses/by-nc-sa/3.0/de/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGC-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGC-1.0.json", - "referenceNumber": 115, - "name": "OGC Software License, Version 1.0", - "licenseId": "OGC-1.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json", + "referenceNumber": 271, + "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + "licenseId": "CC-BY-NC-SA-3.0-IGO", "seeAlso": [ - "https://www.ogc.org/ogc/software/1.0" + "https://creativecommons.org/licenses/by-nc-sa/3.0/igo/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AGPL-3.0-only.html", + "reference": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AGPL-3.0-only.json", - "referenceNumber": 116, - "name": "GNU Affero General Public License v3.0 only", - "licenseId": "AGPL-3.0-only", + "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.json", + "referenceNumber": 434, + "name": "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + "licenseId": "CC-BY-NC-SA-4.0", "seeAlso": [ - "https://www.gnu.org/licenses/agpl.txt", - "https://opensource.org/licenses/AGPL-3.0" + "https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Vim.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Vim.json", - "referenceNumber": 117, - "name": "Vim License", - "licenseId": "Vim", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-1.0.json", + "referenceNumber": 257, + "name": "Creative Commons Attribution No Derivatives 1.0 Generic", + "licenseId": "CC-BY-ND-1.0", "seeAlso": [ - "http://vimdoc.sourceforge.net/htmldoc/uganda.html" + "https://creativecommons.org/licenses/by-nd/1.0/legalcode" ], "isOsiApproved": false, - "isFsfLibre": true + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/ECL-2.0.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ECL-2.0.json", - "referenceNumber": 118, - "name": "Educational Community License v2.0", - "licenseId": "ECL-2.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.0.json", + "referenceNumber": 204, + "name": "Creative Commons Attribution No Derivatives 2.0 Generic", + "licenseId": "CC-BY-ND-2.0", "seeAlso": [ - "https://opensource.org/licenses/ECL-2.0" + "https://creativecommons.org/licenses/by-nd/2.0/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/UCL-1.0.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/UCL-1.0.json", - "referenceNumber": 119, - "name": "Upstream Compatibility License v1.0", - "licenseId": "UCL-1.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.5.json", + "referenceNumber": 72, + "name": "Creative Commons Attribution No Derivatives 2.5 Generic", + "licenseId": "CC-BY-ND-2.5", "seeAlso": [ - "https://opensource.org/licenses/UCL-1.0" + "https://creativecommons.org/licenses/by-nd/2.5/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/mplus.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/mplus.json", - "referenceNumber": 120, - "name": "mplus Font License", - "licenseId": "mplus", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0.json", + "referenceNumber": 331, + "name": "Creative Commons Attribution No Derivatives 3.0 Unported", + "licenseId": "CC-BY-ND-3.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:Mplus?rd\u003dLicensing/mplus" + "https://creativecommons.org/licenses/by-nd/3.0/legalcode" + ], + "isOsiApproved": false, + "isFsfLibre": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.json", + "referenceNumber": 429, + "name": "Creative Commons Attribution No Derivatives 3.0 Germany", + "licenseId": "CC-BY-ND-3.0-DE", + "seeAlso": [ + "https://creativecommons.org/licenses/by-nd/3.0/de/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BlueOak-1.0.0.html", + "reference": "https://spdx.org/licenses/CC-BY-ND-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BlueOak-1.0.0.json", - "referenceNumber": 121, - "name": "Blue Oak Model License 1.0.0", - "licenseId": "BlueOak-1.0.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-4.0.json", + "referenceNumber": 407, + "name": "Creative Commons Attribution No Derivatives 4.0 International", + "licenseId": "CC-BY-ND-4.0", "seeAlso": [ - "https://blueoakcouncil.org/license/1.0.0" + "https://creativecommons.org/licenses/by-nd/4.0/legalcode" + ], + "isOsiApproved": false, + "isFsfLibre": false + }, + { + "reference": "https://spdx.org/licenses/CC-BY-SA-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-1.0.json", + "referenceNumber": 199, + "name": "Creative Commons Attribution Share Alike 1.0 Generic", + "licenseId": "CC-BY-SA-1.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by-sa/1.0/legalcode" ], "isOsiApproved": false }, @@ -1546,7 +1519,7 @@ "reference": "https://spdx.org/licenses/CC-BY-SA-2.0.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.0.json", - "referenceNumber": 122, + "referenceNumber": 478, "name": "Creative Commons Attribution Share Alike 2.0 Generic", "licenseId": "CC-BY-SA-2.0", "seeAlso": [ @@ -1555,1328 +1528,1309 @@ "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AFL-2.1.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-2.1.json", - "referenceNumber": 123, - "name": "Academic Free License v2.1", - "licenseId": "AFL-2.1", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.json", + "referenceNumber": 361, + "name": "Creative Commons Attribution Share Alike 2.0 England and Wales", + "licenseId": "CC-BY-SA-2.0-UK", "seeAlso": [ - "http://opensource.linux-mirror.org/licenses/afl-2.1.txt" + "https://creativecommons.org/licenses/by-sa/2.0/uk/legalcode" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.json", - "referenceNumber": 124, - "name": "GNU General Public License v2.0 w/Classpath exception", - "licenseId": "GPL-2.0-with-classpath-exception", + "reference": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.json", + "referenceNumber": 461, + "name": "Creative Commons Attribution Share Alike 2.1 Japan", + "licenseId": "CC-BY-SA-2.1-JP", "seeAlso": [ - "https://www.gnu.org/software/classpath/license.html" + "https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-1.2.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.2.json", - "referenceNumber": 125, - "name": "Open LDAP Public License v1.2", - "licenseId": "OLDAP-1.2", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.5.json", + "referenceNumber": 418, + "name": "Creative Commons Attribution Share Alike 2.5 Generic", + "licenseId": "CC-BY-SA-2.5", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d42b0383c50c299977b5893ee695cf4e486fb0dc7" + "https://creativecommons.org/licenses/by-sa/2.5/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NGPL.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NGPL.json", - "referenceNumber": 126, - "name": "Nethack General Public License", - "licenseId": "NGPL", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0.json", + "referenceNumber": 340, + "name": "Creative Commons Attribution Share Alike 3.0 Unported", + "licenseId": "CC-BY-SA-3.0", "seeAlso": [ - "https://opensource.org/licenses/NGPL" + "https://creativecommons.org/licenses/by-sa/3.0/legalcode" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/App-s2p.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/App-s2p.json", - "referenceNumber": 127, - "name": "App::s2p License", - "licenseId": "App-s2p", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.json", + "referenceNumber": 260, + "name": "Creative Commons Attribution Share Alike 3.0 Austria", + "licenseId": "CC-BY-SA-3.0-AT", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/App-s2p" + "https://creativecommons.org/licenses/by-sa/3.0/at/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/DL-DE-BY-2.0.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/DL-DE-BY-2.0.json", - "referenceNumber": 128, - "name": "Data licence Germany – attribution – version 2.0", - "licenseId": "DL-DE-BY-2.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-DE.json", + "referenceNumber": 100, + "name": "Creative Commons Attribution Share Alike 3.0 Germany", + "licenseId": "CC-BY-SA-3.0-DE", "seeAlso": [ - "https://www.govdata.de/dl-de/by-2-0" + "https://creativecommons.org/licenses/by-sa/3.0/de/legalcode" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGL-Canada-2.0.html", + "reference": "https://spdx.org/licenses/CC-BY-SA-4.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-Canada-2.0.json", - "referenceNumber": 129, - "name": "Open Government Licence - Canada", - "licenseId": "OGL-Canada-2.0", + "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-4.0.json", + "referenceNumber": 309, + "name": "Creative Commons Attribution Share Alike 4.0 International", + "licenseId": "CC-BY-SA-4.0", "seeAlso": [ - "https://open.canada.ca/en/open-government-licence-canada" + "https://creativecommons.org/licenses/by-sa/4.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.html", + "reference": "https://spdx.org/licenses/CC-PDDC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.json", - "referenceNumber": 130, - "name": "Licence Libre du Québec – Réciprocité forte version 1.1", - "licenseId": "LiLiQ-Rplus-1.1", + "detailsUrl": "https://spdx.org/licenses/CC-PDDC.json", + "referenceNumber": 186, + "name": "Creative Commons Public Domain Dedication and Certification", + "licenseId": "CC-PDDC", "seeAlso": [ - "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-forte-liliq-r-v1-1/", - "http://opensource.org/licenses/LiLiQ-Rplus-1.1" + "https://creativecommons.org/licenses/publicdomain/" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/bzip2-1.0.6.html", + "reference": "https://spdx.org/licenses/CC0-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.6.json", - "referenceNumber": 131, - "name": "bzip2 and libbzip2 License v1.0.6", - "licenseId": "bzip2-1.0.6", + "detailsUrl": "https://spdx.org/licenses/CC0-1.0.json", + "referenceNumber": 249, + "name": "Creative Commons Zero v1.0 Universal", + "licenseId": "CC0-1.0", "seeAlso": [ - "https://sourceware.org/git/?p\u003dbzip2.git;a\u003dblob;f\u003dLICENSE;hb\u003dbzip2-1.0.6", - "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" + "https://creativecommons.org/publicdomain/zero/1.0/legalcode" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/IBM-pibs.html", + "reference": "https://spdx.org/licenses/CDDL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IBM-pibs.json", - "referenceNumber": 132, - "name": "IBM PowerPC Initialization and Boot Software", - "licenseId": "IBM-pibs", + "detailsUrl": "https://spdx.org/licenses/CDDL-1.0.json", + "referenceNumber": 242, + "name": "Common Development and Distribution License 1.0", + "licenseId": "CDDL-1.0", "seeAlso": [ - "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003darch/powerpc/cpu/ppc4xx/miiphy.c;h\u003d297155fdafa064b955e53e9832de93bfb0cfb85b;hb\u003d9fab4bf4cc077c21e43941866f3f2c196f28670d" + "https://opensource.org/licenses/cddl1" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3.json", - "referenceNumber": 133, - "name": "GNU Free Documentation License v1.3", - "licenseId": "GFDL-1.3", + "reference": "https://spdx.org/licenses/CDDL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CDDL-1.1.json", + "referenceNumber": 485, + "name": "Common Development and Distribution License 1.1", + "licenseId": "CDDL-1.1", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "http://glassfish.java.net/public/CDDL+GPL_1_1.html", + "https://javaee.github.io/glassfish/LICENSE" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-1.1.html", + "reference": "https://spdx.org/licenses/CDL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.1.json", - "referenceNumber": 134, - "name": "Open LDAP Public License v1.1", - "licenseId": "OLDAP-1.1", + "detailsUrl": "https://spdx.org/licenses/CDL-1.0.json", + "referenceNumber": 125, + "name": "Common Documentation License 1.0", + "licenseId": "CDL-1.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d806557a5ad59804ef3a44d5abfbe91d706b0791f" + "http://www.opensource.apple.com/cdl/", + "https://fedoraproject.org/wiki/Licensing/Common_Documentation_License", + "https://www.gnu.org/licenses/license-list.html#ACDL" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-only.html", + "reference": "https://spdx.org/licenses/CDLA-Permissive-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-only.json", - "referenceNumber": 135, - "name": "GNU Free Documentation License v1.1 only - invariants", - "licenseId": "GFDL-1.1-invariants-only", + "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-1.0.json", + "referenceNumber": 71, + "name": "Community Data License Agreement Permissive 1.0", + "licenseId": "CDLA-Permissive-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://cdla.io/permissive-1-0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.2.html", + "reference": "https://spdx.org/licenses/CDLA-Permissive-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.json", - "referenceNumber": 136, - "name": "Open LDAP Public License v2.2", - "licenseId": "OLDAP-2.2", + "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-2.0.json", + "referenceNumber": 386, + "name": "Community Data License Agreement Permissive 2.0", + "licenseId": "CDLA-Permissive-2.0", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d470b0c18ec67621c85881b2733057fecf4a1acc3" + "https://cdla.dev/permissive-2-0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NIST-PD.html", + "reference": "https://spdx.org/licenses/CDLA-Sharing-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NIST-PD.json", - "referenceNumber": 137, - "name": "NIST Public Domain Notice", - "licenseId": "NIST-PD", + "detailsUrl": "https://spdx.org/licenses/CDLA-Sharing-1.0.json", + "referenceNumber": 431, + "name": "Community Data License Agreement Sharing 1.0", + "licenseId": "CDLA-Sharing-1.0", "seeAlso": [ - "https://github.com/tcheneau/simpleRPL/blob/e645e69e38dd4e3ccfeceb2db8cba05b7c2e0cd3/LICENSE.txt", - "https://github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md" + "https://cdla.io/sharing-1-0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EUPL-1.0.html", + "reference": "https://spdx.org/licenses/CECILL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUPL-1.0.json", - "referenceNumber": 138, - "name": "European Union Public License 1.0", - "licenseId": "EUPL-1.0", + "detailsUrl": "https://spdx.org/licenses/CECILL-1.0.json", + "referenceNumber": 20, + "name": "CeCILL Free Software License Agreement v1.0", + "licenseId": "CECILL-1.0", "seeAlso": [ - "http://ec.europa.eu/idabc/en/document/7330.html", - "http://ec.europa.eu/idabc/servlets/Doc027f.pdf?id\u003d31096" + "http://www.cecill.info/licences/Licence_CeCILL_V1-fr.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.html", + "reference": "https://spdx.org/licenses/CECILL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.json", - "referenceNumber": 139, - "name": "Mozilla Public License 2.0 (no copyleft exception)", - "licenseId": "MPL-2.0-no-copyleft-exception", + "detailsUrl": "https://spdx.org/licenses/CECILL-1.1.json", + "referenceNumber": 92, + "name": "CeCILL Free Software License Agreement v1.1", + "licenseId": "CECILL-1.1", "seeAlso": [ - "https://www.mozilla.org/MPL/2.0/", - "https://opensource.org/licenses/MPL-2.0" + "http://www.cecill.info/licences/Licence_CeCILL_V1.1-US.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC0-1.0.html", + "reference": "https://spdx.org/licenses/CECILL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC0-1.0.json", - "referenceNumber": 140, - "name": "Creative Commons Zero v1.0 Universal", - "licenseId": "CC0-1.0", + "detailsUrl": "https://spdx.org/licenses/CECILL-2.0.json", + "referenceNumber": 476, + "name": "CeCILL Free Software License Agreement v2.0", + "licenseId": "CECILL-2.0", "seeAlso": [ - "https://creativecommons.org/publicdomain/zero/1.0/legalcode" + "http://www.cecill.info/licences/Licence_CeCILL_V2-en.html" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-US.html", + "reference": "https://spdx.org/licenses/CECILL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-US.json", - "referenceNumber": 141, - "name": "Creative Commons Attribution 3.0 United States", - "licenseId": "CC-BY-3.0-US", + "detailsUrl": "https://spdx.org/licenses/CECILL-2.1.json", + "referenceNumber": 321, + "name": "CeCILL Free Software License Agreement v2.1", + "licenseId": "CECILL-2.1", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/us/legalcode" + "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/EUPL-1.1.html", + "reference": "https://spdx.org/licenses/CECILL-B.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUPL-1.1.json", - "referenceNumber": 142, - "name": "European Union Public License 1.1", - "licenseId": "EUPL-1.1", + "detailsUrl": "https://spdx.org/licenses/CECILL-B.json", + "referenceNumber": 462, + "name": "CeCILL-B Free Software License Agreement", + "licenseId": "CECILL-B", "seeAlso": [ - "https://joinup.ec.europa.eu/software/page/eupl/licence-eupl", - "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl1.1.-licence-en_0.pdf", - "https://opensource.org/licenses/EUPL-1.1" + "http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-3.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0+.json", - "referenceNumber": 143, - "name": "GNU General Public License v3.0 or later", - "licenseId": "GPL-3.0+", + "reference": "https://spdx.org/licenses/CECILL-C.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CECILL-C.json", + "referenceNumber": 360, + "name": "CeCILL-C Free Software License Agreement", + "licenseId": "CECILL-C", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.8.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.8.json", - "referenceNumber": 144, - "name": "Open LDAP Public License v2.8", - "licenseId": "OLDAP-2.8", - "seeAlso": [ - "http://www.openldap.org/software/release/license.html" - ], - "isOsiApproved": true - }, - { - "reference": "https://spdx.org/licenses/OSL-2.1.html", + "reference": "https://spdx.org/licenses/CERN-OHL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-2.1.json", - "referenceNumber": 145, - "name": "Open Software License 2.1", - "licenseId": "OSL-2.1", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.1.json", + "referenceNumber": 56, + "name": "CERN Open Hardware Licence v1.1", + "licenseId": "CERN-OHL-1.1", "seeAlso": [ - "http://web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm", - "https://opensource.org/licenses/OSL-2.1" + "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/libpng-2.0.html", + "reference": "https://spdx.org/licenses/CERN-OHL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/libpng-2.0.json", - "referenceNumber": 146, - "name": "PNG Reference Library version 2", - "licenseId": "libpng-2.0", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.2.json", + "referenceNumber": 189, + "name": "CERN Open Hardware Licence v1.2", + "licenseId": "CERN-OHL-1.2", "seeAlso": [ - "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" + "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/O-UDA-1.0.html", + "reference": "https://spdx.org/licenses/CERN-OHL-P-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/O-UDA-1.0.json", - "referenceNumber": 147, - "name": "Open Use of Data Agreement v1.0", - "licenseId": "O-UDA-1.0", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-P-2.0.json", + "referenceNumber": 122, + "name": "CERN Open Hardware Licence Version 2 - Permissive", + "licenseId": "CERN-OHL-P-2.0", "seeAlso": [ - "https://github.com/microsoft/Open-Use-of-Data-Agreement/blob/v1.0/O-UDA-1.0.md", - "https://cdla.dev/open-use-of-data-agreement-v1-0/" + "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/gnuplot.html", + "reference": "https://spdx.org/licenses/CERN-OHL-S-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/gnuplot.json", - "referenceNumber": 148, - "name": "gnuplot License", - "licenseId": "gnuplot", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-S-2.0.json", + "referenceNumber": 181, + "name": "CERN Open Hardware Licence Version 2 - Strongly Reciprocal", + "licenseId": "CERN-OHL-S-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Gnuplot" + "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Modification.html", + "reference": "https://spdx.org/licenses/CERN-OHL-W-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Modification.json", - "referenceNumber": 149, - "name": "BSD 3-Clause Modification", - "licenseId": "BSD-3-Clause-Modification", + "detailsUrl": "https://spdx.org/licenses/CERN-OHL-W-2.0.json", + "referenceNumber": 213, + "name": "CERN Open Hardware Licence Version 2 - Weakly Reciprocal", + "licenseId": "CERN-OHL-W-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:BSD#Modification_Variant" + "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/ODC-By-1.0.html", + "reference": "https://spdx.org/licenses/checkmk.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ODC-By-1.0.json", - "referenceNumber": 150, - "name": "Open Data Commons Attribution License v1.0", - "licenseId": "ODC-By-1.0", + "detailsUrl": "https://spdx.org/licenses/checkmk.json", + "referenceNumber": 69, + "name": "Checkmk License", + "licenseId": "checkmk", "seeAlso": [ - "https://opendatacommons.org/licenses/by/1.0/" + "https://github.com/libcheck/check/blob/master/checkmk/checkmk.in" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Imlib2.html", + "reference": "https://spdx.org/licenses/ClArtistic.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Imlib2.json", - "referenceNumber": 151, - "name": "Imlib2 License", - "licenseId": "Imlib2", + "detailsUrl": "https://spdx.org/licenses/ClArtistic.json", + "referenceNumber": 209, + "name": "Clarified Artistic License", + "licenseId": "ClArtistic", "seeAlso": [ - "http://trac.enlightenment.org/e/browser/trunk/imlib2/COPYING", - "https://git.enlightenment.org/legacy/imlib2.git/tree/COPYING" + "http://gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/", + "http://www.ncftp.com/ncftp/doc/LICENSE.txt" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OpenSSL.html", + "reference": "https://spdx.org/licenses/CNRI-Jython.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OpenSSL.json", - "referenceNumber": 152, - "name": "OpenSSL License", - "licenseId": "OpenSSL", + "detailsUrl": "https://spdx.org/licenses/CNRI-Jython.json", + "referenceNumber": 108, + "name": "CNRI Jython License", + "licenseId": "CNRI-Jython", "seeAlso": [ - "http://www.openssl.org/source/license.html" + "http://www.jython.org/license.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BUSL-1.1.html", + "reference": "https://spdx.org/licenses/CNRI-Python.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BUSL-1.1.json", - "referenceNumber": 153, - "name": "Business Source License 1.1", - "licenseId": "BUSL-1.1", + "detailsUrl": "https://spdx.org/licenses/CNRI-Python.json", + "referenceNumber": 396, + "name": "CNRI Python License", + "licenseId": "CNRI-Python", "seeAlso": [ - "https://mariadb.com/bsl11/" + "https://opensource.org/licenses/CNRI-Python" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/EUDatagrid.html", + "reference": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EUDatagrid.json", - "referenceNumber": 154, - "name": "EU DataGrid Software License", - "licenseId": "EUDatagrid", + "detailsUrl": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.json", + "referenceNumber": 123, + "name": "CNRI Python Open Source GPL Compatible License Agreement", + "licenseId": "CNRI-Python-GPL-Compatible", "seeAlso": [ - "http://eu-datagrid.web.cern.ch/eu-datagrid/license.html", - "https://opensource.org/licenses/EUDatagrid" + "http://www.python.org/download/releases/1.6.1/download_win/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/EFL-2.0.html", + "reference": "https://spdx.org/licenses/COIL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EFL-2.0.json", - "referenceNumber": 155, - "name": "Eiffel Forum License v2.0", - "licenseId": "EFL-2.0", + "detailsUrl": "https://spdx.org/licenses/COIL-1.0.json", + "referenceNumber": 136, + "name": "Copyfree Open Innovation License", + "licenseId": "COIL-1.0", "seeAlso": [ - "http://www.eiffel-nice.org/license/eiffel-forum-license-2.html", - "https://opensource.org/licenses/EFL-2.0" + "https://coil.apotheon.org/plaintext/01.0.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NRL.html", + "reference": "https://spdx.org/licenses/Community-Spec-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NRL.json", - "referenceNumber": 156, - "name": "NRL License", - "licenseId": "NRL", + "detailsUrl": "https://spdx.org/licenses/Community-Spec-1.0.json", + "referenceNumber": 183, + "name": "Community Specification License 1.0", + "licenseId": "Community-Spec-1.0", "seeAlso": [ - "http://web.mit.edu/network/isakmp/nrllicense.html" + "https://github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OSL-1.1.html", + "reference": "https://spdx.org/licenses/Condor-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-1.1.json", - "referenceNumber": 157, - "name": "Open Software License 1.1", - "licenseId": "OSL-1.1", + "detailsUrl": "https://spdx.org/licenses/Condor-1.1.json", + "referenceNumber": 387, + "name": "Condor Public License v1.1", + "licenseId": "Condor-1.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/OSL1.1" + "http://research.cs.wisc.edu/condor/license.html#condor", + "http://web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/RHeCos-1.1.html", + "reference": "https://spdx.org/licenses/copyleft-next-0.3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RHeCos-1.1.json", - "referenceNumber": 158, - "name": "Red Hat eCos Public License v1.1", - "licenseId": "RHeCos-1.1", + "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.0.json", + "referenceNumber": 44, + "name": "copyleft-next 0.3.0", + "licenseId": "copyleft-next-0.3.0", "seeAlso": [ - "http://ecos.sourceware.org/old-license.html" + "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.0" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/JasPer-2.0.html", + "reference": "https://spdx.org/licenses/copyleft-next-0.3.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/JasPer-2.0.json", - "referenceNumber": 159, - "name": "JasPer License", - "licenseId": "JasPer-2.0", + "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.1.json", + "referenceNumber": 359, + "name": "copyleft-next 0.3.1", + "licenseId": "copyleft-next-0.3.1", "seeAlso": [ - "http://www.ece.uvic.ca/~mdadams/jasper/LICENSE" + "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.html", + "reference": "https://spdx.org/licenses/CPAL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.json", - "referenceNumber": 160, - "name": "GNU Free Documentation License v1.2 only - no invariants", - "licenseId": "GFDL-1.2-no-invariants-only", + "detailsUrl": "https://spdx.org/licenses/CPAL-1.0.json", + "referenceNumber": 467, + "name": "Common Public Attribution License 1.0", + "licenseId": "CPAL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://opensource.org/licenses/CPAL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.html", + "reference": "https://spdx.org/licenses/CPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json", - "referenceNumber": 161, - "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", - "licenseId": "CC-BY-NC-SA-2.0-UK", + "detailsUrl": "https://spdx.org/licenses/CPL-1.0.json", + "referenceNumber": 280, + "name": "Common Public License 1.0", + "licenseId": "CPL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.0/uk/legalcode" + "https://opensource.org/licenses/CPL-1.0" ], - "isOsiApproved": false - }, - { - "reference": "https://spdx.org/licenses/MulanPSL-2.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MulanPSL-2.0.json", - "referenceNumber": 162, - "name": "Mulan Permissive Software License, Version 2", - "licenseId": "MulanPSL-2.0", - "seeAlso": [ - "https://license.coscl.org.cn/MulanPSL2/" - ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/XFree86-1.1.html", + "reference": "https://spdx.org/licenses/CPOL-1.02.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/XFree86-1.1.json", - "referenceNumber": 163, - "name": "XFree86 License 1.1", - "licenseId": "XFree86-1.1", + "detailsUrl": "https://spdx.org/licenses/CPOL-1.02.json", + "referenceNumber": 482, + "name": "Code Project Open License 1.02", + "licenseId": "CPOL-1.02", "seeAlso": [ - "http://www.xfree86.org/current/LICENSE4.html" + "http://www.codeproject.com/info/cpol10.aspx" ], "isOsiApproved": false, - "isFsfLibre": true + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/SMPPL.html", + "reference": "https://spdx.org/licenses/Crossword.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SMPPL.json", - "referenceNumber": 164, - "name": "Secure Messaging Protocol Public License", - "licenseId": "SMPPL", + "detailsUrl": "https://spdx.org/licenses/Crossword.json", + "referenceNumber": 53, + "name": "Crossword License", + "licenseId": "Crossword", "seeAlso": [ - "https://github.com/dcblake/SMP/blob/master/Documentation/License.txt" + "https://fedoraproject.org/wiki/Licensing/Crossword" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDDL-1.1.html", + "reference": "https://spdx.org/licenses/CrystalStacker.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDDL-1.1.json", - "referenceNumber": 165, - "name": "Common Development and Distribution License 1.1", - "licenseId": "CDDL-1.1", + "detailsUrl": "https://spdx.org/licenses/CrystalStacker.json", + "referenceNumber": 141, + "name": "CrystalStacker License", + "licenseId": "CrystalStacker", "seeAlso": [ - "http://glassfish.java.net/public/CDDL+GPL_1_1.html", - "https://javaee.github.io/glassfish/LICENSE" + "https://fedoraproject.org/wiki/Licensing:CrystalStacker?rd\u003dLicensing/CrystalStacker" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/JPNIC.html", + "reference": "https://spdx.org/licenses/CUA-OPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/JPNIC.json", - "referenceNumber": 166, - "name": "Japan Network Information Center License", - "licenseId": "JPNIC", + "detailsUrl": "https://spdx.org/licenses/CUA-OPL-1.0.json", + "referenceNumber": 118, + "name": "CUA Office Public License v1.0", + "licenseId": "CUA-OPL-1.0", "seeAlso": [ - "https://gitlab.isc.org/isc-projects/bind9/blob/master/COPYRIGHT#L366" + "https://opensource.org/licenses/CUA-OPL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CDLA-Permissive-1.0.html", + "reference": "https://spdx.org/licenses/Cube.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-1.0.json", - "referenceNumber": 167, - "name": "Community Data License Agreement Permissive 1.0", - "licenseId": "CDLA-Permissive-1.0", + "detailsUrl": "https://spdx.org/licenses/Cube.json", + "referenceNumber": 232, + "name": "Cube License", + "licenseId": "Cube", "seeAlso": [ - "https://cdla.io/permissive-1-0" + "https://fedoraproject.org/wiki/Licensing/Cube" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/xinetd.html", + "reference": "https://spdx.org/licenses/curl.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/xinetd.json", - "referenceNumber": 168, - "name": "xinetd License", - "licenseId": "xinetd", + "detailsUrl": "https://spdx.org/licenses/curl.json", + "referenceNumber": 346, + "name": "curl License", + "licenseId": "curl", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Xinetd_License" + "https://github.com/bagder/curl/blob/master/COPYING" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/X11-distribute-modifications-variant.html", + "reference": "https://spdx.org/licenses/D-FSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/X11-distribute-modifications-variant.json", - "referenceNumber": 169, - "name": "X11 License Distribution Modification Variant", - "licenseId": "X11-distribute-modifications-variant", + "detailsUrl": "https://spdx.org/licenses/D-FSL-1.0.json", + "referenceNumber": 297, + "name": "Deutsche Freie Software Lizenz", + "licenseId": "D-FSL-1.0", "seeAlso": [ - "https://github.com/mirror/ncurses/blob/master/COPYING" + "http://www.dipp.nrw.de/d-fsl/lizenzen/", + "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/de/D-FSL-1_0_de.txt", + "http://www.dipp.nrw.de/d-fsl/index_html/lizenzen/en/D-FSL-1_0_en.txt", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/deutsche-freie-software-lizenz", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/german-free-software-license", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_de.txt/at_download/file", + "https://www.hbz-nrw.de/produkte/open-access/lizenzen/dfsl/D-FSL-1_0_en.txt/at_download/file" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.3.html", + "reference": "https://spdx.org/licenses/diffmark.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.3.json", - "referenceNumber": 170, - "name": "Open LDAP Public License v2.3", - "licenseId": "OLDAP-2.3", + "detailsUrl": "https://spdx.org/licenses/diffmark.json", + "referenceNumber": 37, + "name": "diffmark license", + "licenseId": "diffmark", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dd32cf54a32d581ab475d23c810b0a7fbaf8d63c3" + "https://fedoraproject.org/wiki/Licensing/diffmark" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SCEA.html", + "reference": "https://spdx.org/licenses/DL-DE-BY-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SCEA.json", - "referenceNumber": 171, - "name": "SCEA Shared Source License", - "licenseId": "SCEA", + "detailsUrl": "https://spdx.org/licenses/DL-DE-BY-2.0.json", + "referenceNumber": 265, + "name": "Data licence Germany – attribution – version 2.0", + "licenseId": "DL-DE-BY-2.0", "seeAlso": [ - "http://research.scea.com/scea_shared_source_license.html" + "https://www.govdata.de/dl-de/by-2-0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SugarCRM-1.1.3.html", + "reference": "https://spdx.org/licenses/DOC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SugarCRM-1.1.3.json", - "referenceNumber": 172, - "name": "SugarCRM Public License v1.1.3", - "licenseId": "SugarCRM-1.1.3", + "detailsUrl": "https://spdx.org/licenses/DOC.json", + "referenceNumber": 453, + "name": "DOC License", + "licenseId": "DOC", "seeAlso": [ - "http://www.sugarcrm.com/crm/SPL" + "http://www.cs.wustl.edu/~schmidt/ACE-copying.html", + "https://www.dre.vanderbilt.edu/~schmidt/ACE-copying.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.0-only.html", + "reference": "https://spdx.org/licenses/Dotseqn.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-only.json", - "referenceNumber": 173, - "name": "GNU Library General Public License v2 only", - "licenseId": "LGPL-2.0-only", + "detailsUrl": "https://spdx.org/licenses/Dotseqn.json", + "referenceNumber": 48, + "name": "Dotseqn License", + "licenseId": "Dotseqn", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + "https://fedoraproject.org/wiki/Licensing/Dotseqn" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDDL-1.0.html", + "reference": "https://spdx.org/licenses/DRL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDDL-1.0.json", - "referenceNumber": 174, - "name": "Common Development and Distribution License 1.0", - "licenseId": "CDDL-1.0", + "detailsUrl": "https://spdx.org/licenses/DRL-1.0.json", + "referenceNumber": 239, + "name": "Detection Rule License 1.0", + "licenseId": "DRL-1.0", "seeAlso": [ - "https://opensource.org/licenses/cddl1" + "https://github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SGI-B-2.0.html", + "reference": "https://spdx.org/licenses/DSDP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SGI-B-2.0.json", - "referenceNumber": 175, - "name": "SGI Free Software License B v2.0", - "licenseId": "SGI-B-2.0", + "detailsUrl": "https://spdx.org/licenses/DSDP.json", + "referenceNumber": 404, + "name": "DSDP License", + "licenseId": "DSDP", "seeAlso": [ - "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf" + "https://fedoraproject.org/wiki/Licensing/DSDP" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.html", + "reference": "https://spdx.org/licenses/dvipdfm.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0.json", - "referenceNumber": 176, - "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", - "licenseId": "CC-BY-NC-SA-3.0", + "detailsUrl": "https://spdx.org/licenses/dvipdfm.json", + "referenceNumber": 388, + "name": "dvipdfm License", + "licenseId": "dvipdfm", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/3.0/legalcode" + "https://fedoraproject.org/wiki/Licensing/dvipdfm" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NetCDF.html", + "reference": "https://spdx.org/licenses/ECL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NetCDF.json", - "referenceNumber": 177, - "name": "NetCDF license", - "licenseId": "NetCDF", + "detailsUrl": "https://spdx.org/licenses/ECL-1.0.json", + "referenceNumber": 298, + "name": "Educational Community License v1.0", + "licenseId": "ECL-1.0", "seeAlso": [ - "http://www.unidata.ucar.edu/software/netcdf/copyright.html" + "https://opensource.org/licenses/ECL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/HaskellReport.html", + "reference": "https://spdx.org/licenses/ECL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/HaskellReport.json", - "referenceNumber": 178, - "name": "Haskell Language Report License", - "licenseId": "HaskellReport", + "detailsUrl": "https://spdx.org/licenses/ECL-2.0.json", + "referenceNumber": 35, + "name": "Educational Community License v2.0", + "licenseId": "ECL-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Haskell_Language_Report_License" + "https://opensource.org/licenses/ECL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LGPL-2.0-or-later.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-or-later.json", - "referenceNumber": 179, - "name": "GNU Library General Public License v2 or later", - "licenseId": "LGPL-2.0-or-later", + "reference": "https://spdx.org/licenses/eCos-2.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/eCos-2.0.json", + "referenceNumber": 285, + "name": "eCos license version 2.0", + "licenseId": "eCos-2.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + "https://www.gnu.org/licenses/ecos-license.html" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.html", + "reference": "https://spdx.org/licenses/EFL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-4.0.json", - "referenceNumber": 180, - "name": "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", - "licenseId": "CC-BY-NC-ND-4.0", + "detailsUrl": "https://spdx.org/licenses/EFL-1.0.json", + "referenceNumber": 238, + "name": "Eiffel Forum License v1.0", + "licenseId": "EFL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode" + "http://www.eiffel-nice.org/license/forum.txt", + "https://opensource.org/licenses/EFL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/BSD-1-Clause.html", + "reference": "https://spdx.org/licenses/EFL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-1-Clause.json", - "referenceNumber": 181, - "name": "BSD 1-Clause License", - "licenseId": "BSD-1-Clause", + "detailsUrl": "https://spdx.org/licenses/EFL-2.0.json", + "referenceNumber": 409, + "name": "Eiffel Forum License v2.0", + "licenseId": "EFL-2.0", "seeAlso": [ - "https://svnweb.freebsd.org/base/head/include/ifaddrs.h?revision\u003d326823" + "http://www.eiffel-nice.org/license/eiffel-forum-license-2.html", + "https://opensource.org/licenses/EFL-2.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/COIL-1.0.html", + "reference": "https://spdx.org/licenses/eGenix.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/COIL-1.0.json", - "referenceNumber": 182, - "name": "Copyfree Open Innovation License", - "licenseId": "COIL-1.0", + "detailsUrl": "https://spdx.org/licenses/eGenix.json", + "referenceNumber": 243, + "name": "eGenix.com Public License 1.1.0", + "licenseId": "eGenix", "seeAlso": [ - "https://coil.apotheon.org/plaintext/01.0.txt" + "http://www.egenix.com/products/eGenix.com-Public-License-1.1.0.pdf", + "https://fedoraproject.org/wiki/Licensing/eGenix.com_Public_License_1.1.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-4.0.html", + "reference": "https://spdx.org/licenses/Elastic-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-4.0.json", - "referenceNumber": 183, - "name": "Creative Commons Attribution No Derivatives 4.0 International", - "licenseId": "CC-BY-ND-4.0", + "detailsUrl": "https://spdx.org/licenses/Elastic-2.0.json", + "referenceNumber": 275, + "name": "Elastic License 2.0", + "licenseId": "Elastic-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/4.0/legalcode" + "https://www.elastic.co/licensing/elastic-license", + "https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE-2.0.txt" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ANTLR-PD.html", + "reference": "https://spdx.org/licenses/Entessa.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ANTLR-PD.json", - "referenceNumber": 184, - "name": "ANTLR Software Rights Notice", - "licenseId": "ANTLR-PD", + "detailsUrl": "https://spdx.org/licenses/Entessa.json", + "referenceNumber": 487, + "name": "Entessa Public License v1.0", + "licenseId": "Entessa", "seeAlso": [ - "http://www.antlr2.org/license.html" + "https://opensource.org/licenses/Entessa" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/BSD-4-Clause-Shortened.html", + "reference": "https://spdx.org/licenses/EPICS.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-Shortened.json", - "referenceNumber": 185, - "name": "BSD 4 Clause Shortened", - "licenseId": "BSD-4-Clause-Shortened", + "detailsUrl": "https://spdx.org/licenses/EPICS.json", + "referenceNumber": 26, + "name": "EPICS Open License", + "licenseId": "EPICS", "seeAlso": [ - "https://metadata.ftp-master.debian.org/changelogs//main/a/arpwatch/arpwatch_2.1a15-7_copyright" + "https://epics.anl.gov/license/open.php" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/diffmark.html", + "reference": "https://spdx.org/licenses/EPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/diffmark.json", - "referenceNumber": 186, - "name": "diffmark license", - "licenseId": "diffmark", + "detailsUrl": "https://spdx.org/licenses/EPL-1.0.json", + "referenceNumber": 456, + "name": "Eclipse Public License 1.0", + "licenseId": "EPL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/diffmark" + "http://www.eclipse.org/legal/epl-v10.html", + "https://opensource.org/licenses/EPL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SSH-OpenSSH.html", + "reference": "https://spdx.org/licenses/EPL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SSH-OpenSSH.json", - "referenceNumber": 187, - "name": "SSH OpenSSH license", - "licenseId": "SSH-OpenSSH", + "detailsUrl": "https://spdx.org/licenses/EPL-2.0.json", + "referenceNumber": 24, + "name": "Eclipse Public License 2.0", + "licenseId": "EPL-2.0", "seeAlso": [ - "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10" + "https://www.eclipse.org/legal/epl-2.0", + "https://www.opensource.org/licenses/EPL-2.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/ErlPL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ErlPL-1.1.json", + "referenceNumber": 109, + "name": "Erlang Public License v1.1", + "licenseId": "ErlPL-1.1", + "seeAlso": [ + "http://www.erlang.org/EPLICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/DOC.html", + "reference": "https://spdx.org/licenses/etalab-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/DOC.json", - "referenceNumber": 188, - "name": "DOC License", - "licenseId": "DOC", + "detailsUrl": "https://spdx.org/licenses/etalab-2.0.json", + "referenceNumber": 484, + "name": "Etalab Open License 2.0", + "licenseId": "etalab-2.0", "seeAlso": [ - "http://www.cs.wustl.edu/~schmidt/ACE-copying.html", - "https://www.dre.vanderbilt.edu/~schmidt/ACE-copying.html" + "https://github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf", + "https://raw.githubusercontent.com/DISIC/politique-de-contribution-open-source/master/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OSL-1.0.html", + "reference": "https://spdx.org/licenses/EUDatagrid.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-1.0.json", - "referenceNumber": 189, - "name": "Open Software License 1.0", - "licenseId": "OSL-1.0", + "detailsUrl": "https://spdx.org/licenses/EUDatagrid.json", + "referenceNumber": 405, + "name": "EU DataGrid Software License", + "licenseId": "EUDatagrid", "seeAlso": [ - "https://opensource.org/licenses/OSL-1.0" + "http://eu-datagrid.web.cern.ch/eu-datagrid/license.html", + "https://opensource.org/licenses/EUDatagrid" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Xnet.html", + "reference": "https://spdx.org/licenses/EUPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Xnet.json", - "referenceNumber": 190, - "name": "X.Net License", - "licenseId": "Xnet", + "detailsUrl": "https://spdx.org/licenses/EUPL-1.0.json", + "referenceNumber": 81, + "name": "European Union Public License 1.0", + "licenseId": "EUPL-1.0", "seeAlso": [ - "https://opensource.org/licenses/Xnet" + "http://ec.europa.eu/idabc/en/document/7330.html", + "http://ec.europa.eu/idabc/servlets/Doc027f.pdf?id\u003d31096" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDL-1.0.html", + "reference": "https://spdx.org/licenses/EUPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDL-1.0.json", - "referenceNumber": 191, - "name": "Common Documentation License 1.0", - "licenseId": "CDL-1.0", + "detailsUrl": "https://spdx.org/licenses/EUPL-1.1.json", + "referenceNumber": 21, + "name": "European Union Public License 1.1", + "licenseId": "EUPL-1.1", "seeAlso": [ - "http://www.opensource.apple.com/cdl/", - "https://fedoraproject.org/wiki/Licensing/Common_Documentation_License", - "https://www.gnu.org/licenses/license-list.html#ACDL" + "https://joinup.ec.europa.eu/software/page/eupl/licence-eupl", + "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl1.1.-licence-en_0.pdf", + "https://opensource.org/licenses/EUPL-1.1" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Latex2e.html", + "reference": "https://spdx.org/licenses/EUPL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Latex2e.json", - "referenceNumber": 192, - "name": "Latex2e License", - "licenseId": "Latex2e", + "detailsUrl": "https://spdx.org/licenses/EUPL-1.2.json", + "referenceNumber": 420, + "name": "European Union Public License 1.2", + "licenseId": "EUPL-1.2", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Latex2e" + "https://joinup.ec.europa.eu/page/eupl-text-11-12", + "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/eupl_v1.2_en.pdf", + "https://joinup.ec.europa.eu/sites/default/files/custom-page/attachment/2020-03/EUPL-1.2%20EN.txt", + "https://joinup.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt", + "http://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri\u003dCELEX:32017D0863", + "https://opensource.org/licenses/EUPL-1.2" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-1.0-or-later.html", + "reference": "https://spdx.org/licenses/Eurosym.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0-or-later.json", - "referenceNumber": 193, - "name": "GNU General Public License v1.0 or later", - "licenseId": "GPL-1.0-or-later", + "detailsUrl": "https://spdx.org/licenses/Eurosym.json", + "referenceNumber": 470, + "name": "Eurosym License", + "licenseId": "Eurosym", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + "https://fedoraproject.org/wiki/Licensing/Eurosym" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ISC.html", + "reference": "https://spdx.org/licenses/Fair.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ISC.json", - "referenceNumber": 194, - "name": "ISC License", - "licenseId": "ISC", + "detailsUrl": "https://spdx.org/licenses/Fair.json", + "referenceNumber": 177, + "name": "Fair License", + "licenseId": "Fair", "seeAlso": [ - "https://www.isc.org/licenses/", - "https://www.isc.org/downloads/software-support-policy/isc-license/", - "https://opensource.org/licenses/ISC" + "http://fairlicense.org/", + "https://opensource.org/licenses/Fair" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Xerox.html", + "reference": "https://spdx.org/licenses/FDK-AAC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Xerox.json", - "referenceNumber": 195, - "name": "Xerox License", - "licenseId": "Xerox", + "detailsUrl": "https://spdx.org/licenses/FDK-AAC.json", + "referenceNumber": 32, + "name": "Fraunhofer FDK AAC Codec Library", + "licenseId": "FDK-AAC", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Xerox" + "https://fedoraproject.org/wiki/Licensing/FDK-AAC", + "https://directory.fsf.org/wiki/License:Fdk" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Artistic-1.0.html", + "reference": "https://spdx.org/licenses/Frameworx-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Artistic-1.0.json", - "referenceNumber": 196, - "name": "Artistic License 1.0", - "licenseId": "Artistic-1.0", - "seeAlso": [ - "https://opensource.org/licenses/Artistic-1.0" - ], - "isOsiApproved": true, - "isFsfLibre": false - }, - { - "reference": "https://spdx.org/licenses/CUA-OPL-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CUA-OPL-1.0.json", - "referenceNumber": 197, - "name": "CUA Office Public License v1.0", - "licenseId": "CUA-OPL-1.0", + "detailsUrl": "https://spdx.org/licenses/Frameworx-1.0.json", + "referenceNumber": 46, + "name": "Frameworx Open License 1.0", + "licenseId": "Frameworx-1.0", "seeAlso": [ - "https://opensource.org/licenses/CUA-OPL-1.0" + "https://opensource.org/licenses/Frameworx-1.0" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/NPL-1.1.html", + "reference": "https://spdx.org/licenses/FreeBSD-DOC.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NPL-1.1.json", - "referenceNumber": 198, - "name": "Netscape Public License v1.1", - "licenseId": "NPL-1.1", + "detailsUrl": "https://spdx.org/licenses/FreeBSD-DOC.json", + "referenceNumber": 40, + "name": "FreeBSD Documentation License", + "licenseId": "FreeBSD-DOC", "seeAlso": [ - "http://www.mozilla.org/MPL/NPL/1.1/" + "https://www.freebsd.org/copyright/freebsd-doc-license/" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MITNFA.html", + "reference": "https://spdx.org/licenses/FreeImage.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MITNFA.json", - "referenceNumber": 199, - "name": "MIT +no-false-attribs license", - "licenseId": "MITNFA", + "detailsUrl": "https://spdx.org/licenses/FreeImage.json", + "referenceNumber": 130, + "name": "FreeImage Public License v1.0", + "licenseId": "FreeImage", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MITNFA" + "http://freeimage.sourceforge.net/freeimage-license.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-1.4.html", + "reference": "https://spdx.org/licenses/FSFAP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.4.json", - "referenceNumber": 200, - "name": "Open LDAP Public License v1.4", - "licenseId": "OLDAP-1.4", + "detailsUrl": "https://spdx.org/licenses/FSFAP.json", + "referenceNumber": 120, + "name": "FSF All Permissive License", + "licenseId": "FSFAP", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dc9f95c2f3f2ffb5e0ae55fe7388af75547660941" + "https://www.gnu.org/prep/maintain/html_node/License-Notices-for-Other-Files.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Leptonica.html", + "reference": "https://spdx.org/licenses/FSFUL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Leptonica.json", - "referenceNumber": 201, - "name": "Leptonica License", - "licenseId": "Leptonica", + "detailsUrl": "https://spdx.org/licenses/FSFUL.json", + "referenceNumber": 215, + "name": "FSF Unlimited License", + "licenseId": "FSFUL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Leptonica" + "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OCCT-PL.html", + "reference": "https://spdx.org/licenses/FSFULLR.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OCCT-PL.json", - "referenceNumber": 202, - "name": "Open CASCADE Technology Public License", - "licenseId": "OCCT-PL", + "detailsUrl": "https://spdx.org/licenses/FSFULLR.json", + "referenceNumber": 43, + "name": "FSF Unlimited License (with License Retention)", + "licenseId": "FSFULLR", "seeAlso": [ - "http://www.opencascade.com/content/occt-public-license" + "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Qhull.html", + "reference": "https://spdx.org/licenses/FSFULLRWD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Qhull.json", - "referenceNumber": 203, - "name": "Qhull License", - "licenseId": "Qhull", + "detailsUrl": "https://spdx.org/licenses/FSFULLRWD.json", + "referenceNumber": 253, + "name": "FSF Unlimited License (With License Retention and Warranty Disclaimer)", + "licenseId": "FSFULLRWD", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Qhull" + "https://lists.gnu.org/archive/html/autoconf/2012-04/msg00061.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CPL-1.0.html", + "reference": "https://spdx.org/licenses/FTL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CPL-1.0.json", - "referenceNumber": 204, - "name": "Common Public License 1.0", - "licenseId": "CPL-1.0", + "detailsUrl": "https://spdx.org/licenses/FTL.json", + "referenceNumber": 390, + "name": "Freetype Project License", + "licenseId": "FTL", "seeAlso": [ - "https://opensource.org/licenses/CPL-1.0" + "http://freetype.fis.uniroma2.it/FTL.TXT", + "http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT", + "http://gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.html", + "reference": "https://spdx.org/licenses/GD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License.json", - "referenceNumber": 205, - "name": "BSD 3-Clause No Nuclear License", - "licenseId": "BSD-3-Clause-No-Nuclear-License", + "detailsUrl": "https://spdx.org/licenses/GD.json", + "referenceNumber": 483, + "name": "GD License", + "licenseId": "GD", "seeAlso": [ - "http://download.oracle.com/otn-pub/java/licenses/bsd.txt?AuthParam\u003d1467140197_43d516ce1776bd08a58235a7785be1cc" + "https://libgd.github.io/manuals/2.3.0/files/license-txt.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-2.5-AU.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-2.5-AU.json", - "referenceNumber": 206, - "name": "Creative Commons Attribution 2.5 Australia", - "licenseId": "CC-BY-2.5-AU", + "reference": "https://spdx.org/licenses/GFDL-1.1.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1.json", + "referenceNumber": 355, + "name": "GNU Free Documentation License v1.1", + "licenseId": "GFDL-1.1", "seeAlso": [ - "https://creativecommons.org/licenses/by/2.5/au/legalcode" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.1.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.1.json", - "referenceNumber": 207, - "name": "Open LDAP Public License v2.1", - "licenseId": "OLDAP-2.1", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-only.json", + "referenceNumber": 22, + "name": "GNU Free Documentation License v1.1 only - invariants", + "licenseId": "GFDL-1.1-invariants-only", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db0d176738e96a0d3b9f85cb51e140a86f21be715" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-only.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-only.json", - "referenceNumber": 208, - "name": "GNU General Public License v2.0 only", - "licenseId": "GPL-2.0-only", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.json", + "referenceNumber": 322, + "name": "GNU Free Documentation License v1.1 or later - invariants", + "licenseId": "GFDL-1.1-invariants-or-later", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ImageMagick.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ImageMagick.json", - "referenceNumber": 209, - "name": "ImageMagick License", - "licenseId": "ImageMagick", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-only.json", + "referenceNumber": 450, + "name": "GNU Free Documentation License v1.1 only - no invariants", + "licenseId": "GFDL-1.1-no-invariants-only", "seeAlso": [ - "http://www.imagemagick.org/script/license.php" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-1.0.json", - "referenceNumber": 210, - "name": "Creative Commons Attribution Share Alike 1.0 Generic", - "licenseId": "CC-BY-SA-1.0", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.json", + "referenceNumber": 317, + "name": "GNU Free Documentation License v1.1 or later - no invariants", + "licenseId": "GFDL-1.1-no-invariants-or-later", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/1.0/legalcode" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPL-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPL-1.0.json", - "referenceNumber": 211, - "name": "Lucent Public License Version 1.0", - "licenseId": "LPL-1.0", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-only.json", + "referenceNumber": 0, + "name": "GNU Free Documentation License v1.1 only", + "licenseId": "GFDL-1.1-only", "seeAlso": [ - "https://opensource.org/licenses/LPL-1.0" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NCSA.html", + "reference": "https://spdx.org/licenses/GFDL-1.1-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NCSA.json", - "referenceNumber": 212, - "name": "University of Illinois/NCSA Open Source License", - "licenseId": "NCSA", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-or-later.json", + "referenceNumber": 156, + "name": "GNU Free Documentation License v1.1 or later", + "licenseId": "GFDL-1.1-or-later", "seeAlso": [ - "http://otm.illinois.edu/uiuc_openSource", - "https://opensource.org/licenses/NCSA" + "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause.json", - "referenceNumber": 213, - "name": "BSD 3-Clause \"New\" or \"Revised\" License", - "licenseId": "BSD-3-Clause", + "reference": "https://spdx.org/licenses/GFDL-1.2.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2.json", + "referenceNumber": 299, + "name": "GNU Free Documentation License v1.2", + "licenseId": "GFDL-1.2", "seeAlso": [ - "https://opensource.org/licenses/BSD-3-Clause" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/APSL-1.1.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-1.1.json", - "referenceNumber": 214, - "name": "Apple Public Source License 1.1", - "licenseId": "APSL-1.1", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-only.json", + "referenceNumber": 473, + "name": "GNU Free Documentation License v1.2 only - invariants", + "licenseId": "GFDL-1.2-invariants-only", "seeAlso": [ - "http://www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Python-2.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Python-2.0.json", - "referenceNumber": 215, - "name": "Python License 2.0", - "licenseId": "Python-2.0", - "seeAlso": [ - "https://opensource.org/licenses/Python-2.0" - ], - "isOsiApproved": true, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/RPL-1.1.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RPL-1.1.json", - "referenceNumber": 216, - "name": "Reciprocal Public License 1.1", - "licenseId": "RPL-1.1", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.json", + "referenceNumber": 460, + "name": "GNU Free Documentation License v1.2 or later - invariants", + "licenseId": "GFDL-1.2-invariants-or-later", "seeAlso": [ - "https://opensource.org/licenses/RPL-1.1" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CPOL-1.02.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CPOL-1.02.json", - "referenceNumber": 217, - "name": "Code Project Open License 1.02", - "licenseId": "CPOL-1.02", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-only.json", + "referenceNumber": 195, + "name": "GNU Free Documentation License v1.2 only - no invariants", + "licenseId": "GFDL-1.2-no-invariants-only", "seeAlso": [ - "http://www.codeproject.com/info/cpol10.aspx" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/YPL-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/YPL-1.0.json", - "referenceNumber": 218, - "name": "Yahoo! Public License v1.0", - "licenseId": "YPL-1.0", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-no-invariants-or-later.json", + "referenceNumber": 219, + "name": "GNU Free Documentation License v1.2 or later - no invariants", + "licenseId": "GFDL-1.2-no-invariants-or-later", "seeAlso": [ - "http://www.zimbra.com/license/yahoo_public_license_1.0.html" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AMDPLPA.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AMDPLPA.json", - "referenceNumber": 219, - "name": "AMD\u0027s plpa_map.c License", - "licenseId": "AMDPLPA", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-only.json", + "referenceNumber": 391, + "name": "GNU Free Documentation License v1.2 only", + "licenseId": "GFDL-1.2-only", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AMD_plpa_map_License" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-3.0-only.html", + "reference": "https://spdx.org/licenses/GFDL-1.2-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-only.json", - "referenceNumber": 220, - "name": "GNU General Public License v3.0 only", - "licenseId": "GPL-3.0-only", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-or-later.json", + "referenceNumber": 323, + "name": "GNU Free Documentation License v1.2 or later", + "licenseId": "GFDL-1.2-or-later", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-or-later.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-or-later.json", - "referenceNumber": 221, - "name": "GNU Free Documentation License v1.1 or later", - "licenseId": "GFDL-1.1-or-later", + "reference": "https://spdx.org/licenses/GFDL-1.3.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3.json", + "referenceNumber": 480, + "name": "GNU Free Documentation License v1.3", + "licenseId": "GFDL-1.3", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Info-ZIP.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Info-ZIP.json", - "referenceNumber": 222, - "name": "Info-ZIP License", - "licenseId": "Info-ZIP", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-only.json", + "referenceNumber": 262, + "name": "GNU Free Documentation License v1.3 only - invariants", + "licenseId": "GFDL-1.3-invariants-only", "seeAlso": [ - "http://www.info-zip.org/license.html" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGDL-Taiwan-1.0.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGDL-Taiwan-1.0.json", - "referenceNumber": 223, - "name": "Taiwan Open Government Data License, version 1.0", - "licenseId": "OGDL-Taiwan-1.0", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.json", + "referenceNumber": 190, + "name": "GNU Free Documentation License v1.3 or later - invariants", + "licenseId": "GFDL-1.3-invariants-or-later", "seeAlso": [ - "https://data.gov.tw/license" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Unicode-DFS-2015.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2015.json", - "referenceNumber": 224, - "name": "Unicode License Agreement - Data Files and Software (2015)", - "licenseId": "Unicode-DFS-2015", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-only.json", + "referenceNumber": 344, + "name": "GNU Free Documentation License v1.3 only - no invariants", + "licenseId": "GFDL-1.3-no-invariants-only", "seeAlso": [ - "https://web.archive.org/web/20151224134844/http://unicode.org/copyright.html" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Python-2.0.1.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Python-2.0.1.json", - "referenceNumber": 225, - "name": "Python License 2.0.1", - "licenseId": "Python-2.0.1", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.json", + "referenceNumber": 292, + "name": "GNU Free Documentation License v1.3 or later - no invariants", + "licenseId": "GFDL-1.3-no-invariants-or-later", "seeAlso": [ - "https://www.python.org/download/releases/2.0.1/license/", - "https://docs.python.org/3/license.html", - "https://github.com/python/cpython/blob/main/LICENSE" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-NetBSD.json", - "referenceNumber": 226, - "name": "BSD 2-Clause NetBSD License", - "licenseId": "BSD-2-Clause-NetBSD", + "reference": "https://spdx.org/licenses/GFDL-1.3-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-only.json", + "referenceNumber": 284, + "name": "GNU Free Documentation License v1.3 only", + "licenseId": "GFDL-1.3-only", "seeAlso": [ - "http://www.netbsd.org/about/redistribution.html#default" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LGPL-2.1-only.html", + "reference": "https://spdx.org/licenses/GFDL-1.3-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-only.json", - "referenceNumber": 227, - "name": "GNU Lesser General Public License v2.1 only", - "licenseId": "LGPL-2.1-only", + "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-or-later.json", + "referenceNumber": 435, + "name": "GNU Free Documentation License v1.3 or later", + "licenseId": "GFDL-1.3-or-later", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", - "https://opensource.org/licenses/LGPL-2.1" + "https://www.gnu.org/licenses/fdl-1.3.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, + { + "reference": "https://spdx.org/licenses/Giftware.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Giftware.json", + "referenceNumber": 358, + "name": "Giftware License", + "licenseId": "Giftware", + "seeAlso": [ + "http://liballeg.org/license.html#allegro-4-the-giftware-license" + ], + "isOsiApproved": false + }, { "reference": "https://spdx.org/licenses/GL2PS.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/GL2PS.json", - "referenceNumber": 228, + "referenceNumber": 179, "name": "GL2PS License", "licenseId": "GL2PS", "seeAlso": [ @@ -2885,407 +2839,381 @@ "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TU-Berlin-1.0.html", + "reference": "https://spdx.org/licenses/Glide.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TU-Berlin-1.0.json", - "referenceNumber": 229, - "name": "Technische Universitaet Berlin License 1.0", - "licenseId": "TU-Berlin-1.0", + "detailsUrl": "https://spdx.org/licenses/Glide.json", + "referenceNumber": 119, + "name": "3dfx Glide License", + "licenseId": "Glide", "seeAlso": [ - "https://github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT" + "http://www.users.on.net/~triforce/glidexp/COPYING.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/DSDP.html", + "reference": "https://spdx.org/licenses/Glulxe.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/DSDP.json", - "referenceNumber": 230, - "name": "DSDP License", - "licenseId": "DSDP", + "detailsUrl": "https://spdx.org/licenses/Glulxe.json", + "referenceNumber": 465, + "name": "Glulxe License", + "licenseId": "Glulxe", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/DSDP" + "https://fedoraproject.org/wiki/Licensing/Glulxe" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.html", + "reference": "https://spdx.org/licenses/GLWTPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-or-later.json", - "referenceNumber": 231, - "name": "GNU Free Documentation License v1.3 or later - invariants", - "licenseId": "GFDL-1.3-invariants-or-later", + "detailsUrl": "https://spdx.org/licenses/GLWTPL.json", + "referenceNumber": 11, + "name": "Good Luck With That Public License", + "licenseId": "GLWTPL", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://github.com/me-shaon/GLWTPL/commit/da5f6bc734095efbacb442c0b31e33a65b9d6e85" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Unlicense.html", + "reference": "https://spdx.org/licenses/gnuplot.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unlicense.json", - "referenceNumber": 232, - "name": "The Unlicense", - "licenseId": "Unlicense", + "detailsUrl": "https://spdx.org/licenses/gnuplot.json", + "referenceNumber": 170, + "name": "gnuplot License", + "licenseId": "gnuplot", "seeAlso": [ - "https://unlicense.org/" + "https://fedoraproject.org/wiki/Licensing/Gnuplot" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.2.html", + "reference": "https://spdx.org/licenses/GPL-1.0.html", "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2.json", - "referenceNumber": 233, - "name": "GNU Free Documentation License v1.2", - "licenseId": "GFDL-1.2", + "detailsUrl": "https://spdx.org/licenses/GPL-1.0.json", + "referenceNumber": 489, + "name": "GNU General Public License v1.0 only", + "licenseId": "GPL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BitTorrent-1.1.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.1.json", - "referenceNumber": 234, - "name": "BitTorrent Open Source License v1.1", - "licenseId": "BitTorrent-1.1", - "seeAlso": [ - "http://directory.fsf.org/wiki/License:BitTorrentOSL1.1" - ], - "isOsiApproved": false, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/TCP-wrappers.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TCP-wrappers.json", - "referenceNumber": 235, - "name": "TCP Wrappers License", - "licenseId": "TCP-wrappers", + "reference": "https://spdx.org/licenses/GPL-1.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-1.0+.json", + "referenceNumber": 13, + "name": "GNU General Public License v1.0 or later", + "licenseId": "GPL-1.0+", "seeAlso": [ - "http://rc.quest.com/topics/openssh/license.php#tcpwrappers" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/psutils.html", + "reference": "https://spdx.org/licenses/GPL-1.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/psutils.json", - "referenceNumber": 236, - "name": "psutils License", - "licenseId": "psutils", + "detailsUrl": "https://spdx.org/licenses/GPL-1.0-only.json", + "referenceNumber": 287, + "name": "GNU General Public License v1.0 only", + "licenseId": "GPL-1.0-only", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/psutils" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Abstyles.html", + "reference": "https://spdx.org/licenses/GPL-1.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Abstyles.json", - "referenceNumber": 237, - "name": "Abstyles License", - "licenseId": "Abstyles", + "detailsUrl": "https://spdx.org/licenses/GPL-1.0-or-later.json", + "referenceNumber": 428, + "name": "GNU General Public License v1.0 or later", + "licenseId": "GPL-1.0-or-later", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Abstyles" + "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Plexus.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Plexus.json", - "referenceNumber": 238, - "name": "Plexus Classworlds License", - "licenseId": "Plexus", + "reference": "https://spdx.org/licenses/GPL-2.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0.json", + "referenceNumber": 161, + "name": "GNU General Public License v2.0 only", + "licenseId": "GPL-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Plexus_Classworlds_License" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MIT-0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-0.json", - "referenceNumber": 239, - "name": "MIT No Attribution", - "licenseId": "MIT-0", + "reference": "https://spdx.org/licenses/GPL-2.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0+.json", + "referenceNumber": 115, + "name": "GNU General Public License v2.0 or later", + "licenseId": "GPL-2.0+", "seeAlso": [ - "https://github.com/aws/mit-0", - "https://romanrm.net/mit-zero", - "https://github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], - "isOsiApproved": true + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Zend-2.0.html", + "reference": "https://spdx.org/licenses/GPL-2.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zend-2.0.json", - "referenceNumber": 240, - "name": "Zend License v2.0", - "licenseId": "Zend-2.0", + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-only.json", + "referenceNumber": 25, + "name": "GNU General Public License v2.0 only", + "licenseId": "GPL-2.0-only", "seeAlso": [ - "https://web.archive.org/web/20130517195954/http://www.zend.com/license/2_00.txt" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-or-later.html", + "reference": "https://spdx.org/licenses/GPL-2.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-or-later.json", - "referenceNumber": 241, - "name": "GNU Free Documentation License v1.3 or later", - "licenseId": "GFDL-1.3-or-later", + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-or-later.json", + "referenceNumber": 474, + "name": "GNU General Public License v2.0 or later", + "licenseId": "GPL-2.0-or-later", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", + "https://opensource.org/licenses/GPL-2.0" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.1-JP.json", - "referenceNumber": 242, - "name": "Creative Commons Attribution Share Alike 2.1 Japan", - "licenseId": "CC-BY-SA-2.1-JP", + "reference": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-autoconf-exception.json", + "referenceNumber": 314, + "name": "GNU General Public License v2.0 w/Autoconf exception", + "licenseId": "GPL-2.0-with-autoconf-exception", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode" + "http://ac-archive.sourceforge.net/doc/copyright.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-NL.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-NL.json", - "referenceNumber": 243, - "name": "Creative Commons Attribution 3.0 Netherlands", - "licenseId": "CC-BY-3.0-NL", + "reference": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.json", + "referenceNumber": 374, + "name": "GNU General Public License v2.0 w/Bison exception", + "licenseId": "GPL-2.0-with-bison-exception", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/nl/legalcode" + "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.0-UK.json", - "referenceNumber": 244, - "name": "Creative Commons Attribution Share Alike 2.0 England and Wales", - "licenseId": "CC-BY-SA-2.0-UK", + "reference": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-classpath-exception.json", + "referenceNumber": 15, + "name": "GNU General Public License v2.0 w/Classpath exception", + "licenseId": "GPL-2.0-with-classpath-exception", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/2.0/uk/legalcode" + "https://www.gnu.org/software/classpath/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/eCos-2.0.html", + "reference": "https://spdx.org/licenses/GPL-2.0-with-font-exception.html", "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/eCos-2.0.json", - "referenceNumber": 245, - "name": "eCos license version 2.0", - "licenseId": "eCos-2.0", + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-font-exception.json", + "referenceNumber": 198, + "name": "GNU General Public License v2.0 w/Font exception", + "licenseId": "GPL-2.0-with-font-exception", "seeAlso": [ - "https://www.gnu.org/licenses/ecos-license.html" + "https://www.gnu.org/licenses/gpl-faq.html#FontException" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Elastic-2.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Elastic-2.0.json", - "referenceNumber": 246, - "name": "Elastic License 2.0", - "licenseId": "Elastic-2.0", + "reference": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.json", + "referenceNumber": 131, + "name": "GNU General Public License v2.0 w/GCC Runtime Library exception", + "licenseId": "GPL-2.0-with-GCC-exception", "seeAlso": [ - "https://www.elastic.co/licensing/elastic-license", - "https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE-2.0.txt" + "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Nunit.html", + "reference": "https://spdx.org/licenses/GPL-3.0.html", "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/Nunit.json", - "referenceNumber": 247, - "name": "Nunit License", - "licenseId": "Nunit", + "detailsUrl": "https://spdx.org/licenses/GPL-3.0.json", + "referenceNumber": 171, + "name": "GNU General Public License v3.0 only", + "licenseId": "GPL-3.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Nunit" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SISSL-1.2.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SISSL-1.2.json", - "referenceNumber": 248, - "name": "Sun Industry Standards Source License v1.2", - "licenseId": "SISSL-1.2", + "reference": "https://spdx.org/licenses/GPL-3.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0+.json", + "referenceNumber": 349, + "name": "GNU General Public License v3.0 or later", + "licenseId": "GPL-3.0+", "seeAlso": [ - "http://gridscheduler.sourceforge.net/Gridengine_SISSL_license.html" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Attribution.html", + "reference": "https://spdx.org/licenses/GPL-3.0-only.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Attribution.json", - "referenceNumber": 249, - "name": "BSD with attribution", - "licenseId": "BSD-3-Clause-Attribution", + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-only.json", + "referenceNumber": 471, + "name": "GNU General Public License v3.0 only", + "licenseId": "GPL-3.0-only", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/BSD_with_Attribution" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/EPL-2.0.html", + "reference": "https://spdx.org/licenses/GPL-3.0-or-later.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EPL-2.0.json", - "referenceNumber": 250, - "name": "Eclipse Public License 2.0", - "licenseId": "EPL-2.0", + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-or-later.json", + "referenceNumber": 394, + "name": "GNU General Public License v3.0 or later", + "licenseId": "GPL-3.0-or-later", "seeAlso": [ - "https://www.eclipse.org/legal/epl-2.0", - "https://www.opensource.org/licenses/EPL-2.0" + "https://www.gnu.org/licenses/gpl-3.0-standalone.html", + "https://opensource.org/licenses/GPL-3.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-invariants-or-later.json", - "referenceNumber": 251, - "name": "GNU Free Documentation License v1.1 or later - invariants", - "licenseId": "GFDL-1.1-invariants-or-later", - "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" - ], + "reference": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-autoconf-exception.json", + "referenceNumber": 401, + "name": "GNU General Public License v3.0 w/Autoconf exception", + "licenseId": "GPL-3.0-with-autoconf-exception", + "seeAlso": [ + "https://www.gnu.org/licenses/autoconf-exception-3.0.html" + ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0-or-later.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-or-later.json", - "referenceNumber": 252, - "name": "GNU General Public License v3.0 or later", - "licenseId": "GPL-3.0-or-later", + "reference": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.json", + "referenceNumber": 148, + "name": "GNU General Public License v3.0 w/GCC Runtime Library exception", + "licenseId": "GPL-3.0-with-GCC-exception", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-3.0-standalone.html", - "https://opensource.org/licenses/GPL-3.0" + "https://www.gnu.org/licenses/gcc-exception-3.1.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.html", + "reference": "https://spdx.org/licenses/Graphics-Gems.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CAL-1.0-Combined-Work-Exception.json", - "referenceNumber": 253, - "name": "Cryptographic Autonomy License 1.0 (Combined Work Exception)", - "licenseId": "CAL-1.0-Combined-Work-Exception", + "detailsUrl": "https://spdx.org/licenses/Graphics-Gems.json", + "referenceNumber": 247, + "name": "Graphics Gems License", + "licenseId": "Graphics-Gems", "seeAlso": [ - "http://cryptographicautonomylicense.com/license-text.html", - "https://opensource.org/licenses/CAL-1.0" + "https://github.com/erich666/GraphicsGems/blob/master/LICENSE.md" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPPL-1.2.html", + "reference": "https://spdx.org/licenses/gSOAP-1.3b.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.2.json", - "referenceNumber": 254, - "name": "LaTeX Project Public License v1.2", - "licenseId": "LPPL-1.2", + "detailsUrl": "https://spdx.org/licenses/gSOAP-1.3b.json", + "referenceNumber": 305, + "name": "gSOAP Public License v1.3b", + "licenseId": "gSOAP-1.3b", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-2.txt" + "http://www.cs.fsu.edu/~engelen/license.html" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CNRI-Jython.html", + "reference": "https://spdx.org/licenses/HaskellReport.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CNRI-Jython.json", - "referenceNumber": 255, - "name": "CNRI Jython License", - "licenseId": "CNRI-Jython", + "detailsUrl": "https://spdx.org/licenses/HaskellReport.json", + "referenceNumber": 59, + "name": "Haskell Language Report License", + "licenseId": "HaskellReport", "seeAlso": [ - "http://www.jython.org/license.html" + "https://fedoraproject.org/wiki/Licensing/Haskell_Language_Report_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-4.0.html", + "reference": "https://spdx.org/licenses/Hippocratic-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-4.0.json", - "referenceNumber": 256, - "name": "Creative Commons Attribution Non Commercial 4.0 International", - "licenseId": "CC-BY-NC-4.0", + "detailsUrl": "https://spdx.org/licenses/Hippocratic-2.1.json", + "referenceNumber": 290, + "name": "Hippocratic License 2.1", + "licenseId": "Hippocratic-2.1", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/4.0/legalcode" + "https://firstdonoharm.dev/version/2/1/license.html", + "https://github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AFL-1.1.html", + "reference": "https://spdx.org/licenses/HPND.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-1.1.json", - "referenceNumber": 257, - "name": "Academic Free License v1.1", - "licenseId": "AFL-1.1", + "detailsUrl": "https://spdx.org/licenses/HPND.json", + "referenceNumber": 426, + "name": "Historical Permission Notice and Disclaimer", + "licenseId": "HPND", "seeAlso": [ - "http://opensource.linux-mirror.org/licenses/afl-1.1.txt", - "http://wayback.archive.org/web/20021004124254/http://www.opensource.org/licenses/academic.php" + "https://opensource.org/licenses/HPND" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/ANTLR-PD-fallback.html", + "reference": "https://spdx.org/licenses/HPND-export-US.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ANTLR-PD-fallback.json", - "referenceNumber": 258, - "name": "ANTLR Software Rights Notice with license fallback", - "licenseId": "ANTLR-PD-fallback", + "detailsUrl": "https://spdx.org/licenses/HPND-export-US.json", + "referenceNumber": 319, + "name": "HPND with US Government export control warning", + "licenseId": "HPND-export-US", "seeAlso": [ - "http://www.antlr2.org/license.html" + "https://www.kermitproject.org/ck90.html#source" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AFL-1.2.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-1.2.json", - "referenceNumber": 259, - "name": "Academic Free License v1.2", - "licenseId": "AFL-1.2", - "seeAlso": [ - "http://opensource.linux-mirror.org/licenses/afl-1.2.txt", - "http://wayback.archive.org/web/20021204204652/http://www.opensource.org/licenses/academic.php" - ], - "isOsiApproved": true, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/NLOD-1.0.html", + "reference": "https://spdx.org/licenses/HPND-sell-variant.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NLOD-1.0.json", - "referenceNumber": 260, - "name": "Norwegian Licence for Open Government Data (NLOD) 1.0", - "licenseId": "NLOD-1.0", + "detailsUrl": "https://spdx.org/licenses/HPND-sell-variant.json", + "referenceNumber": 272, + "name": "Historical Permission Notice and Disclaimer - sell variant", + "licenseId": "HPND-sell-variant", "seeAlso": [ - "http://data.norge.no/nlod/en/1.0" + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h\u003dv4.19" ], "isOsiApproved": false }, @@ -3293,7 +3221,7 @@ "reference": "https://spdx.org/licenses/HTMLTIDY.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/HTMLTIDY.json", - "referenceNumber": 261, + "referenceNumber": 303, "name": "HTML Tidy License", "licenseId": "HTMLTIDY", "seeAlso": [ @@ -3302,965 +3230,1330 @@ "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-DE.html", + "reference": "https://spdx.org/licenses/IBM-pibs.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-DE.json", - "referenceNumber": 262, - "name": "Creative Commons Attribution 3.0 Germany", - "licenseId": "CC-BY-3.0-DE", + "detailsUrl": "https://spdx.org/licenses/IBM-pibs.json", + "referenceNumber": 498, + "name": "IBM PowerPC Initialization and Boot Software", + "licenseId": "IBM-pibs", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/de/legalcode" + "http://git.denx.de/?p\u003du-boot.git;a\u003dblob;f\u003darch/powerpc/cpu/ppc4xx/miiphy.c;h\u003d297155fdafa064b955e53e9832de93bfb0cfb85b;hb\u003d9fab4bf4cc077c21e43941866f3f2c196f28670d" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ECL-1.0.html", + "reference": "https://spdx.org/licenses/ICU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ECL-1.0.json", - "referenceNumber": 263, - "name": "Educational Community License v1.0", - "licenseId": "ECL-1.0", + "detailsUrl": "https://spdx.org/licenses/ICU.json", + "referenceNumber": 197, + "name": "ICU License", + "licenseId": "ICU", "seeAlso": [ - "https://opensource.org/licenses/ECL-1.0" + "http://source.icu-project.org/repos/icu/icu/trunk/license.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.html", + "reference": "https://spdx.org/licenses/IJG.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-License-2014.json", - "referenceNumber": 264, - "name": "BSD 3-Clause No Nuclear License 2014", - "licenseId": "BSD-3-Clause-No-Nuclear-License-2014", + "detailsUrl": "https://spdx.org/licenses/IJG.json", + "referenceNumber": 38, + "name": "Independent JPEG Group License", + "licenseId": "IJG", "seeAlso": [ - "https://java.net/projects/javaeetutorial/pages/BerkeleyLicense" + "http://dev.w3.org/cvsweb/Amaya/libjpeg/Attic/README?rev\u003d1.2" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OFL-1.0-no-RFN.html", + "reference": "https://spdx.org/licenses/IJG-short.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.0-no-RFN.json", - "referenceNumber": 265, - "name": "SIL Open Font License 1.0 with no Reserved Font Name", - "licenseId": "OFL-1.0-no-RFN", + "detailsUrl": "https://spdx.org/licenses/IJG-short.json", + "referenceNumber": 281, + "name": "Independent JPEG Group License - short", + "licenseId": "IJG-short", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" + "https://sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.html", + "reference": "https://spdx.org/licenses/ImageMagick.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-or-later.json", - "referenceNumber": 266, - "name": "GNU Free Documentation License v1.2 or later - invariants", - "licenseId": "GFDL-1.2-invariants-or-later", + "detailsUrl": "https://spdx.org/licenses/ImageMagick.json", + "referenceNumber": 488, + "name": "ImageMagick License", + "licenseId": "ImageMagick", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "http://www.imagemagick.org/script/license.php" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CECILL-B.html", + "reference": "https://spdx.org/licenses/iMatix.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-B.json", - "referenceNumber": 267, - "name": "CeCILL-B Free Software License Agreement", - "licenseId": "CECILL-B", + "detailsUrl": "https://spdx.org/licenses/iMatix.json", + "referenceNumber": 449, + "name": "iMatix Standard Function Library Agreement", + "licenseId": "iMatix", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.html" + "http://legacy.imatix.com/html/sfl/sfl4.htm#license" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CECILL-2.1.html", + "reference": "https://spdx.org/licenses/Imlib2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-2.1.json", - "referenceNumber": 268, - "name": "CeCILL Free Software License Agreement v2.1", - "licenseId": "CECILL-2.1", + "detailsUrl": "https://spdx.org/licenses/Imlib2.json", + "referenceNumber": 258, + "name": "Imlib2 License", + "licenseId": "Imlib2", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.html" + "http://trac.enlightenment.org/e/browser/trunk/imlib2/COPYING", + "https://git.enlightenment.org/legacy/imlib2.git/tree/COPYING" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SGI-B-1.0.html", + "reference": "https://spdx.org/licenses/Info-ZIP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SGI-B-1.0.json", - "referenceNumber": 269, - "name": "SGI Free Software License B v1.0", - "licenseId": "SGI-B-1.0", + "detailsUrl": "https://spdx.org/licenses/Info-ZIP.json", + "referenceNumber": 245, + "name": "Info-ZIP License", + "licenseId": "Info-ZIP", "seeAlso": [ - "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html" + "http://www.info-zip.org/license.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NBPL-1.0.html", + "reference": "https://spdx.org/licenses/Intel.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NBPL-1.0.json", - "referenceNumber": 270, - "name": "Net Boolean Public License v1", - "licenseId": "NBPL-1.0", + "detailsUrl": "https://spdx.org/licenses/Intel.json", + "referenceNumber": 423, + "name": "Intel Open Source License", + "licenseId": "Intel", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d37b4b3f6cc4bf34e1d3dec61e69914b9819d8894" + "https://opensource.org/licenses/Intel" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.html", + "reference": "https://spdx.org/licenses/Intel-ACPI.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CNRI-Python-GPL-Compatible.json", - "referenceNumber": 271, - "name": "CNRI Python Open Source GPL Compatible License Agreement", - "licenseId": "CNRI-Python-GPL-Compatible", + "detailsUrl": "https://spdx.org/licenses/Intel-ACPI.json", + "referenceNumber": 373, + "name": "Intel ACPI Software License Agreement", + "licenseId": "Intel-ACPI", "seeAlso": [ - "http://www.python.org/download/releases/1.6.1/download_win/" + "https://fedoraproject.org/wiki/Licensing/Intel_ACPI_Software_License_Agreement" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SchemeReport.html", + "reference": "https://spdx.org/licenses/Interbase-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SchemeReport.json", - "referenceNumber": 272, - "name": "Scheme Language Report License", - "licenseId": "SchemeReport", - "seeAlso": [], + "detailsUrl": "https://spdx.org/licenses/Interbase-1.0.json", + "referenceNumber": 505, + "name": "Interbase Public License v1.0", + "licenseId": "Interbase-1.0", + "seeAlso": [ + "https://web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html" + ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Apache-2.0.html", + "reference": "https://spdx.org/licenses/IPA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Apache-2.0.json", - "referenceNumber": 273, - "name": "Apache License 2.0", - "licenseId": "Apache-2.0", + "detailsUrl": "https://spdx.org/licenses/IPA.json", + "referenceNumber": 234, + "name": "IPA Font License", + "licenseId": "IPA", + "seeAlso": [ + "https://opensource.org/licenses/IPA" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/IPL-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/IPL-1.0.json", + "referenceNumber": 447, + "name": "IBM Public License v1.0", + "licenseId": "IPL-1.0", + "seeAlso": [ + "https://opensource.org/licenses/IPL-1.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/ISC.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ISC.json", + "referenceNumber": 62, + "name": "ISC License", + "licenseId": "ISC", + "seeAlso": [ + "https://www.isc.org/licenses/", + "https://www.isc.org/downloads/software-support-policy/isc-license/", + "https://opensource.org/licenses/ISC" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/Jam.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Jam.json", + "referenceNumber": 135, + "name": "Jam License", + "licenseId": "Jam", + "seeAlso": [ + "https://www.boost.org/doc/libs/1_35_0/doc/html/jam.html", + "https://web.archive.org/web/20160330173339/https://swarm.workshop.perforce.com/files/guest/perforce_software/jam/src/README" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/JasPer-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/JasPer-2.0.json", + "referenceNumber": 289, + "name": "JasPer License", + "licenseId": "JasPer-2.0", + "seeAlso": [ + "http://www.ece.uvic.ca/~mdadams/jasper/LICENSE" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/JPNIC.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/JPNIC.json", + "referenceNumber": 440, + "name": "Japan Network Information Center License", + "licenseId": "JPNIC", + "seeAlso": [ + "https://gitlab.isc.org/isc-projects/bind9/blob/master/COPYRIGHT#L366" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/JSON.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/JSON.json", + "referenceNumber": 174, + "name": "JSON License", + "licenseId": "JSON", + "seeAlso": [ + "http://www.json.org/license.html" + ], + "isOsiApproved": false, + "isFsfLibre": false + }, + { + "reference": "https://spdx.org/licenses/Knuth-CTAN.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Knuth-CTAN.json", + "referenceNumber": 76, + "name": "Knuth CTAN License", + "licenseId": "Knuth-CTAN", + "seeAlso": [ + "https://ctan.org/license/knuth" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LAL-1.2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LAL-1.2.json", + "referenceNumber": 185, + "name": "Licence Art Libre 1.2", + "licenseId": "LAL-1.2", + "seeAlso": [ + "http://artlibre.org/licence/lal/licence-art-libre-12/" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LAL-1.3.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LAL-1.3.json", + "referenceNumber": 362, + "name": "Licence Art Libre 1.3", + "licenseId": "LAL-1.3", + "seeAlso": [ + "https://artlibre.org/" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Latex2e.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Latex2e.json", + "referenceNumber": 231, + "name": "Latex2e License", + "licenseId": "Latex2e", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Latex2e" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Leptonica.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Leptonica.json", + "referenceNumber": 427, + "name": "Leptonica License", + "licenseId": "Leptonica", + "seeAlso": [ + "https://fedoraproject.org/wiki/Licensing/Leptonica" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0.json", + "referenceNumber": 495, + "name": "GNU Library General Public License v2 only", + "licenseId": "LGPL-2.0", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0+.json", + "referenceNumber": 216, + "name": "GNU Library General Public License v2 or later", + "licenseId": "LGPL-2.0+", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.0-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-only.json", + "referenceNumber": 83, + "name": "GNU Library General Public License v2 only", + "licenseId": "LGPL-2.0-only", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.0-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.0-or-later.json", + "referenceNumber": 27, + "name": "GNU Library General Public License v2 or later", + "licenseId": "LGPL-2.0-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.1.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1.json", + "referenceNumber": 503, + "name": "GNU Lesser General Public License v2.1 only", + "licenseId": "LGPL-2.1", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.1+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1+.json", + "referenceNumber": 436, + "name": "GNU Lesser General Public License v2.1 or later", + "licenseId": "LGPL-2.1+", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.1-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-only.json", + "referenceNumber": 172, + "name": "GNU Lesser General Public License v2.1 only", + "licenseId": "LGPL-2.1-only", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-2.1-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-or-later.json", + "referenceNumber": 367, + "name": "GNU Lesser General Public License v2.1 or later", + "licenseId": "LGPL-2.1-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", + "https://opensource.org/licenses/LGPL-2.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-3.0.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0.json", + "referenceNumber": 506, + "name": "GNU Lesser General Public License v3.0 only", + "licenseId": "LGPL-3.0", + "seeAlso": [ + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-3.0+.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0+.json", + "referenceNumber": 176, + "name": "GNU Lesser General Public License v3.0 or later", + "licenseId": "LGPL-3.0+", + "seeAlso": [ + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-3.0-only.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-only.json", + "referenceNumber": 313, + "name": "GNU Lesser General Public License v3.0 only", + "licenseId": "LGPL-3.0-only", + "seeAlso": [ + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPL-3.0-or-later.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPL-3.0-or-later.json", + "referenceNumber": 113, + "name": "GNU Lesser General Public License v3.0 or later", + "licenseId": "LGPL-3.0-or-later", + "seeAlso": [ + "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", + "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", + "https://opensource.org/licenses/LGPL-3.0" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/LGPLLR.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LGPLLR.json", + "referenceNumber": 137, + "name": "Lesser General Public License For Linguistic Resources", + "licenseId": "LGPLLR", + "seeAlso": [ + "http://www-igm.univ-mlv.fr/~unitex/lgpllr.html" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/Libpng.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Libpng.json", + "referenceNumber": 392, + "name": "libpng License", + "licenseId": "Libpng", + "seeAlso": [ + "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/libpng-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/libpng-2.0.json", + "referenceNumber": 308, + "name": "PNG Reference Library version 2", + "licenseId": "libpng-2.0", + "seeAlso": [ + "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" + ], + "isOsiApproved": false + }, + { + "reference": "https://spdx.org/licenses/libselinux-1.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/libselinux-1.0.json", + "referenceNumber": 372, + "name": "libselinux public domain notice", + "licenseId": "libselinux-1.0", "seeAlso": [ - "https://www.apache.org/licenses/LICENSE-2.0", - "https://opensource.org/licenses/Apache-2.0" + "https://github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ODbL-1.0.html", + "reference": "https://spdx.org/licenses/libtiff.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ODbL-1.0.json", - "referenceNumber": 274, - "name": "Open Data Commons Open Database License v1.0", - "licenseId": "ODbL-1.0", + "detailsUrl": "https://spdx.org/licenses/libtiff.json", + "referenceNumber": 240, + "name": "libtiff License", + "licenseId": "libtiff", "seeAlso": [ - "http://www.opendatacommons.org/licenses/odbl/1.0/", - "https://opendatacommons.org/licenses/odbl/1-0/" + "https://fedoraproject.org/wiki/Licensing/libtiff" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-3.0.html", + "reference": "https://spdx.org/licenses/libutil-David-Nugent.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0.json", - "referenceNumber": 275, - "name": "Creative Commons Attribution No Derivatives 3.0 Unported", - "licenseId": "CC-BY-ND-3.0", + "detailsUrl": "https://spdx.org/licenses/libutil-David-Nugent.json", + "referenceNumber": 150, + "name": "libutil David Nugent License", + "licenseId": "libutil-David-Nugent", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/3.0/legalcode" + "http://web.mit.edu/freebsd/head/lib/libutil/login_ok.3", + "https://cgit.freedesktop.org/libbsd/tree/man/setproctitle.3bsd" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPPL-1.3c.html", + "reference": "https://spdx.org/licenses/LiLiQ-P-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.3c.json", - "referenceNumber": 276, - "name": "LaTeX Project Public License v1.3c", - "licenseId": "LPPL-1.3c", + "detailsUrl": "https://spdx.org/licenses/LiLiQ-P-1.1.json", + "referenceNumber": 430, + "name": "Licence Libre du Québec – Permissive version 1.1", + "licenseId": "LiLiQ-P-1.1", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-3c.txt", - "https://opensource.org/licenses/LPPL-1.3c" + "https://forge.gouv.qc.ca/licence/fr/liliq-v1-1/", + "http://opensource.org/licenses/LiLiQ-P-1.1" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/APSL-1.2.html", + "reference": "https://spdx.org/licenses/LiLiQ-R-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-1.2.json", - "referenceNumber": 277, - "name": "Apple Public Source License 1.2", - "licenseId": "APSL-1.2", + "detailsUrl": "https://spdx.org/licenses/LiLiQ-R-1.1.json", + "referenceNumber": 266, + "name": "Licence Libre du Québec – Réciprocité version 1.1", + "licenseId": "LiLiQ-R-1.1", "seeAlso": [ - "http://www.samurajdata.se/opensource/mirror/licenses/apsl.php" + "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-liliq-r-v1-1/", + "http://opensource.org/licenses/LiLiQ-R-1.1" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OFL-1.1.html", + "reference": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.1.json", - "referenceNumber": 278, - "name": "SIL Open Font License 1.1", - "licenseId": "OFL-1.1", + "detailsUrl": "https://spdx.org/licenses/LiLiQ-Rplus-1.1.json", + "referenceNumber": 211, + "name": "Licence Libre du Québec – Réciprocité forte version 1.1", + "licenseId": "LiLiQ-Rplus-1.1", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", - "https://opensource.org/licenses/OFL-1.1" + "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-forte-liliq-r-v1-1/", + "http://opensource.org/licenses/LiLiQ-Rplus-1.1" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/MS-PL.html", + "reference": "https://spdx.org/licenses/Linux-man-pages-copyleft.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MS-PL.json", - "referenceNumber": 279, - "name": "Microsoft Public License", - "licenseId": "MS-PL", + "detailsUrl": "https://spdx.org/licenses/Linux-man-pages-copyleft.json", + "referenceNumber": 3, + "name": "Linux man-pages Copyleft", + "licenseId": "Linux-man-pages-copyleft", "seeAlso": [ - "http://www.microsoft.com/opensource/licenses.mspx", - "https://opensource.org/licenses/MS-PL" + "https://www.kernel.org/doc/man-pages/licenses.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/C-UDA-1.0.html", + "reference": "https://spdx.org/licenses/Linux-OpenIB.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/C-UDA-1.0.json", - "referenceNumber": 280, - "name": "Computational Use of Data Agreement v1.0", - "licenseId": "C-UDA-1.0", + "detailsUrl": "https://spdx.org/licenses/Linux-OpenIB.json", + "referenceNumber": 175, + "name": "Linux Kernel Variant of OpenIB.org license", + "licenseId": "Linux-OpenIB", "seeAlso": [ - "https://github.com/microsoft/Computational-Use-of-Data-Agreement/blob/master/C-UDA-1.0.md", - "https://cdla.dev/computational-use-of-data-agreement-v1-0/" + "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Sendmail.html", + "reference": "https://spdx.org/licenses/LOOP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Sendmail.json", - "referenceNumber": 281, - "name": "Sendmail License", - "licenseId": "Sendmail", + "detailsUrl": "https://spdx.org/licenses/LOOP.json", + "referenceNumber": 325, + "name": "Common Lisp LOOP License", + "licenseId": "LOOP", "seeAlso": [ - "http://www.sendmail.com/pdfs/open_source/sendmail_license.pdf", - "https://web.archive.org/web/20160322142305/https://www.sendmail.com/pdfs/open_source/sendmail_license.pdf" + "https://gitlab.com/embeddable-common-lisp/ecl/-/blob/develop/src/lsp/loop.lsp", + "http://git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h\u003dVersion_2_6_13pre", + "https://sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp", + "https://github.com/cl-adams/adams/blob/master/LICENSE.md", + "https://github.com/blakemcbride/eclipse-lisp/blob/master/lisp/loop.lisp", + "https://gitlab.common-lisp.net/cmucl/cmucl/-/blob/master/src/code/loop.lisp" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.html", + "reference": "https://spdx.org/licenses/LPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json", - "referenceNumber": 282, - "name": "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", - "licenseId": "CC-BY-NC-SA-3.0-DE", + "detailsUrl": "https://spdx.org/licenses/LPL-1.0.json", + "referenceNumber": 307, + "name": "Lucent Public License Version 1.0", + "licenseId": "LPL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/3.0/de/legalcode" + "https://opensource.org/licenses/LPL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/GPL-2.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0.json", - "referenceNumber": 283, - "name": "GNU General Public License v2.0 only", - "licenseId": "GPL-2.0", + "reference": "https://spdx.org/licenses/LPL-1.02.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/LPL-1.02.json", + "referenceNumber": 339, + "name": "Lucent Public License v1.02", + "licenseId": "LPL-1.02", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "http://plan9.bell-labs.com/plan9/license.html", + "https://opensource.org/licenses/LPL-1.02" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/libselinux-1.0.html", + "reference": "https://spdx.org/licenses/LPPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/libselinux-1.0.json", - "referenceNumber": 284, - "name": "libselinux public domain notice", - "licenseId": "libselinux-1.0", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.0.json", + "referenceNumber": 252, + "name": "LaTeX Project Public License v1.0", + "licenseId": "LPPL-1.0", "seeAlso": [ - "https://github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE" + "http://www.latex-project.org/lppl/lppl-1-0.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-2.5.html", + "reference": "https://spdx.org/licenses/LPPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.5.json", - "referenceNumber": 285, - "name": "Creative Commons Attribution No Derivatives 2.5 Generic", - "licenseId": "CC-BY-ND-2.5", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.1.json", + "referenceNumber": 366, + "name": "LaTeX Project Public License v1.1", + "licenseId": "LPPL-1.1", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/2.5/legalcode" + "http://www.latex-project.org/lppl/lppl-1-1.txt" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-3.0.html", + "reference": "https://spdx.org/licenses/LPPL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-3.0.json", - "referenceNumber": 286, - "name": "Creative Commons Attribution Non Commercial 3.0 Unported", - "licenseId": "CC-BY-NC-3.0", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.2.json", + "referenceNumber": 327, + "name": "LaTeX Project Public License v1.2", + "licenseId": "LPPL-1.2", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/3.0/legalcode" + "http://www.latex-project.org/lppl/lppl-1-2.txt" ], "isOsiApproved": false, - "isFsfLibre": false + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MS-LPL.html", + "reference": "https://spdx.org/licenses/LPPL-1.3a.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MS-LPL.json", - "referenceNumber": 287, - "name": "Microsoft Limited Public License", - "licenseId": "MS-LPL", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.3a.json", + "referenceNumber": 64, + "name": "LaTeX Project Public License v1.3a", + "licenseId": "LPPL-1.3a", "seeAlso": [ - "https://www.openhub.net/licenses/mslpl", - "https://github.com/gabegundy/atlserver/blob/master/License.txt", - "https://en.wikipedia.org/wiki/Shared_Source_Initiative#Microsoft_Limited_Public_License_(Ms-LPL)" + "http://www.latex-project.org/lppl/lppl-1-3a.txt" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.html", + "reference": "https://spdx.org/licenses/LPPL-1.3c.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0.json", - "referenceNumber": 288, - "name": "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", - "licenseId": "CC-BY-NC-SA-2.0", + "detailsUrl": "https://spdx.org/licenses/LPPL-1.3c.json", + "referenceNumber": 19, + "name": "LaTeX Project Public License v1.3c", + "licenseId": "LPPL-1.3c", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.0/legalcode" + "http://www.latex-project.org/lppl/lppl-1-3c.txt", + "https://opensource.org/licenses/LPPL-1.3c" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/AMPAS.html", + "reference": "https://spdx.org/licenses/LZMA-SDK-9.11-to-9.20.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AMPAS.json", - "referenceNumber": 289, - "name": "Academy of Motion Picture Arts and Sciences BSD", - "licenseId": "AMPAS", + "detailsUrl": "https://spdx.org/licenses/LZMA-SDK-9.11-to-9.20.json", + "referenceNumber": 96, + "name": "LZMA SDK License (versions 9.11 to 9.20)", + "licenseId": "LZMA-SDK-9.11-to-9.20", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/BSD#AMPASBSD" + "https://www.7-zip.org/sdk.html", + "https://sourceforge.net/projects/sevenzip/files/LZMA%20SDK/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AAL.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AAL.json", - "referenceNumber": 290, - "name": "Attribution Assurance License", - "licenseId": "AAL", - "seeAlso": [ - "https://opensource.org/licenses/attribution" - ], - "isOsiApproved": true - }, - { - "reference": "https://spdx.org/licenses/Bahyph.html", + "reference": "https://spdx.org/licenses/LZMA-SDK-9.22.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Bahyph.json", - "referenceNumber": 291, - "name": "Bahyph License", - "licenseId": "Bahyph", + "detailsUrl": "https://spdx.org/licenses/LZMA-SDK-9.22.json", + "referenceNumber": 31, + "name": "LZMA SDK License (versions 9.22 and beyond)", + "licenseId": "LZMA-SDK-9.22", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Bahyph" + "https://www.7-zip.org/sdk.html", + "https://sourceforge.net/projects/sevenzip/files/LZMA%20SDK/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Entessa.html", + "reference": "https://spdx.org/licenses/MakeIndex.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Entessa.json", - "referenceNumber": 292, - "name": "Entessa Public License v1.0", - "licenseId": "Entessa", + "detailsUrl": "https://spdx.org/licenses/MakeIndex.json", + "referenceNumber": 158, + "name": "MakeIndex License", + "licenseId": "MakeIndex", "seeAlso": [ - "https://opensource.org/licenses/Entessa" + "https://fedoraproject.org/wiki/Licensing/MakeIndex" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0-IGO.html", + "reference": "https://spdx.org/licenses/Minpack.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0-IGO.json", - "referenceNumber": 293, - "name": "Creative Commons Attribution 3.0 IGO", - "licenseId": "CC-BY-3.0-IGO", + "detailsUrl": "https://spdx.org/licenses/Minpack.json", + "referenceNumber": 508, + "name": "Minpack License", + "licenseId": "Minpack", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/igo/legalcode" + "http://www.netlib.org/minpack/disclaimer", + "https://gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.html", + "reference": "https://spdx.org/licenses/MirOS.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-no-invariants-or-later.json", - "referenceNumber": 294, - "name": "GNU Free Documentation License v1.3 or later - no invariants", - "licenseId": "GFDL-1.3-no-invariants-or-later", + "detailsUrl": "https://spdx.org/licenses/MirOS.json", + "referenceNumber": 479, + "name": "The MirOS Licence", + "licenseId": "MirOS", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://opensource.org/licenses/MirOS" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/IJG.html", + "reference": "https://spdx.org/licenses/MIT.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IJG.json", - "referenceNumber": 295, - "name": "Independent JPEG Group License", - "licenseId": "IJG", + "detailsUrl": "https://spdx.org/licenses/MIT.json", + "referenceNumber": 111, + "name": "MIT License", + "licenseId": "MIT", "seeAlso": [ - "http://dev.w3.org/cvsweb/Amaya/libjpeg/Attic/README?rev\u003d1.2" + "https://opensource.org/licenses/MIT" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CERN-OHL-W-2.0.html", + "reference": "https://spdx.org/licenses/MIT-0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-W-2.0.json", - "referenceNumber": 296, - "name": "CERN Open Hardware Licence Version 2 - Weakly Reciprocal", - "licenseId": "CERN-OHL-W-2.0", + "detailsUrl": "https://spdx.org/licenses/MIT-0.json", + "referenceNumber": 320, + "name": "MIT No Attribution", + "licenseId": "MIT-0", "seeAlso": [ - "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + "https://github.com/aws/mit-0", + "https://romanrm.net/mit-zero", + "https://github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/SWL.html", + "reference": "https://spdx.org/licenses/MIT-advertising.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SWL.json", - "referenceNumber": 297, - "name": "Scheme Widget Library (SWL) Software License Agreement", - "licenseId": "SWL", + "detailsUrl": "https://spdx.org/licenses/MIT-advertising.json", + "referenceNumber": 75, + "name": "Enlightenment License (e16)", + "licenseId": "MIT-advertising", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/SWL" + "https://fedoraproject.org/wiki/Licensing/MIT_With_Advertising" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FSFULLR.html", + "reference": "https://spdx.org/licenses/MIT-CMU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FSFULLR.json", - "referenceNumber": 298, - "name": "FSF Unlimited License (with License Retention)", - "licenseId": "FSFULLR", + "detailsUrl": "https://spdx.org/licenses/MIT-CMU.json", + "referenceNumber": 164, + "name": "CMU License", + "licenseId": "MIT-CMU", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/FSF_Unlimited_License#License_Retention_Variant" + "https://fedoraproject.org/wiki/Licensing:MIT?rd\u003dLicensing/MIT#CMU_Style", + "https://github.com/python-pillow/Pillow/blob/fffb426092c8db24a5f4b6df243a8a3c01fb63cd/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/TMate.html", + "reference": "https://spdx.org/licenses/MIT-enna.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TMate.json", - "referenceNumber": 299, - "name": "TMate Open Source License", - "licenseId": "TMate", + "detailsUrl": "https://spdx.org/licenses/MIT-enna.json", + "referenceNumber": 121, + "name": "enna License", + "licenseId": "MIT-enna", "seeAlso": [ - "http://svnkit.com/license.html" + "https://fedoraproject.org/wiki/Licensing/MIT#enna" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Artistic-2.0.html", + "reference": "https://spdx.org/licenses/MIT-feh.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Artistic-2.0.json", - "referenceNumber": 300, - "name": "Artistic License 2.0", - "licenseId": "Artistic-2.0", + "detailsUrl": "https://spdx.org/licenses/MIT-feh.json", + "referenceNumber": 145, + "name": "feh License", + "licenseId": "MIT-feh", "seeAlso": [ - "http://www.perlfoundation.org/artistic_license_2_0", - "https://www.perlfoundation.org/artistic-license-20.html", - "https://opensource.org/licenses/artistic-license-2.0" + "https://fedoraproject.org/wiki/Licensing/MIT#feh" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/APSL-1.0.html", + "reference": "https://spdx.org/licenses/MIT-Modern-Variant.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-1.0.json", - "referenceNumber": 301, - "name": "Apple Public Source License 1.0", - "licenseId": "APSL-1.0", + "detailsUrl": "https://spdx.org/licenses/MIT-Modern-Variant.json", + "referenceNumber": 12, + "name": "MIT License Modern Variant", + "licenseId": "MIT-Modern-Variant", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Apple_Public_Source_License_1.0" + "https://fedoraproject.org/wiki/Licensing:MIT#Modern_Variants", + "https://ptolemy.berkeley.edu/copyright.htm", + "https://pirlwww.lpl.arizona.edu/resources/guide/software/PerlTk/Tixlic.html" ], - "isOsiApproved": true, - "isFsfLibre": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/ClArtistic.html", + "reference": "https://spdx.org/licenses/MIT-open-group.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ClArtistic.json", - "referenceNumber": 302, - "name": "Clarified Artistic License", - "licenseId": "ClArtistic", + "detailsUrl": "https://spdx.org/licenses/MIT-open-group.json", + "referenceNumber": 276, + "name": "MIT Open Group variant", + "licenseId": "MIT-open-group", "seeAlso": [ - "http://gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/", - "http://www.ncftp.com/ncftp/doc/LICENSE.txt" + "https://gitlab.freedesktop.org/xorg/app/iceauth/-/blob/master/COPYING", + "https://gitlab.freedesktop.org/xorg/app/xvinfo/-/blob/master/COPYING", + "https://gitlab.freedesktop.org/xorg/app/xsetroot/-/blob/master/COPYING", + "https://gitlab.freedesktop.org/xorg/app/xauth/-/blob/master/COPYING" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ZPL-1.1.html", + "reference": "https://spdx.org/licenses/MIT-Wu.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ZPL-1.1.json", - "referenceNumber": 303, - "name": "Zope Public License 1.1", - "licenseId": "ZPL-1.1", + "detailsUrl": "https://spdx.org/licenses/MIT-Wu.json", + "referenceNumber": 415, + "name": "MIT Tom Wu Variant", + "licenseId": "MIT-Wu", "seeAlso": [ - "http://old.zope.org/Resources/License/ZPL-1.1" + "https://github.com/chromium/octane/blob/master/crypto.js" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NLPL.html", + "reference": "https://spdx.org/licenses/MITNFA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NLPL.json", - "referenceNumber": 304, - "name": "No Limit Public License", - "licenseId": "NLPL", + "detailsUrl": "https://spdx.org/licenses/MITNFA.json", + "referenceNumber": 73, + "name": "MIT +no-false-attribs license", + "licenseId": "MITNFA", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/NLPL" + "https://fedoraproject.org/wiki/Licensing/MITNFA" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OSL-2.0.html", + "reference": "https://spdx.org/licenses/Motosoto.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-2.0.json", - "referenceNumber": 305, - "name": "Open Software License 2.0", - "licenseId": "OSL-2.0", + "detailsUrl": "https://spdx.org/licenses/Motosoto.json", + "referenceNumber": 267, + "name": "Motosoto License", + "licenseId": "Motosoto", "seeAlso": [ - "http://web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html" + "https://opensource.org/licenses/Motosoto" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/psfrag.html", + "reference": "https://spdx.org/licenses/mpi-permissive.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/psfrag.json", - "referenceNumber": 306, - "name": "psfrag License", - "licenseId": "psfrag", + "detailsUrl": "https://spdx.org/licenses/mpi-permissive.json", + "referenceNumber": 235, + "name": "mpi Permissive License", + "licenseId": "mpi-permissive", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/psfrag" + "https://sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl\u003d19#L19" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/mpi-permissive.html", + "reference": "https://spdx.org/licenses/mpich2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/mpi-permissive.json", - "referenceNumber": 307, - "name": "mpi Permissive License", - "licenseId": "mpi-permissive", + "detailsUrl": "https://spdx.org/licenses/mpich2.json", + "referenceNumber": 246, + "name": "mpich2 License", + "licenseId": "mpich2", "seeAlso": [ - "https://sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl\u003d19#L19" + "https://fedoraproject.org/wiki/Licensing/MIT" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.html", + "reference": "https://spdx.org/licenses/MPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-1.0.json", - "referenceNumber": 308, - "name": "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", - "licenseId": "CC-BY-NC-ND-1.0", + "detailsUrl": "https://spdx.org/licenses/MPL-1.0.json", + "referenceNumber": 79, + "name": "Mozilla Public License 1.0", + "licenseId": "MPL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd-nc/1.0/legalcode" + "http://www.mozilla.org/MPL/MPL-1.0.html", + "https://opensource.org/licenses/MPL-1.0" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/MPL-1.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MPL-1.1.json", + "referenceNumber": 10, + "name": "Mozilla Public License 1.1", + "licenseId": "MPL-1.1", + "seeAlso": [ + "http://www.mozilla.org/MPL/MPL-1.1.html", + "https://opensource.org/licenses/MPL-1.1" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/MPL-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MPL-2.0.json", + "referenceNumber": 438, + "name": "Mozilla Public License 2.0", + "licenseId": "MPL-2.0", + "seeAlso": [ + "https://www.mozilla.org/MPL/2.0/", + "https://opensource.org/licenses/MPL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.2.1.html", + "reference": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.1.json", - "referenceNumber": 309, - "name": "Open LDAP Public License v2.2.1", - "licenseId": "OLDAP-2.2.1", + "detailsUrl": "https://spdx.org/licenses/MPL-2.0-no-copyleft-exception.json", + "referenceNumber": 210, + "name": "Mozilla Public License 2.0 (no copyleft exception)", + "licenseId": "MPL-2.0-no-copyleft-exception", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d4bc786f34b50aa301be6f5600f58a980070f481e" + "https://www.mozilla.org/MPL/2.0/", + "https://opensource.org/licenses/MPL-2.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/SSPL-1.0.html", + "reference": "https://spdx.org/licenses/mplus.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SSPL-1.0.json", - "referenceNumber": 310, - "name": "Server Side Public License, v 1", - "licenseId": "SSPL-1.0", + "detailsUrl": "https://spdx.org/licenses/mplus.json", + "referenceNumber": 336, + "name": "mplus Font License", + "licenseId": "mplus", "seeAlso": [ - "https://www.mongodb.com/licensing/server-side-public-license" + "https://fedoraproject.org/wiki/Licensing:Mplus?rd\u003dLicensing/mplus" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Dotseqn.html", + "reference": "https://spdx.org/licenses/MS-LPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Dotseqn.json", - "referenceNumber": 311, - "name": "Dotseqn License", - "licenseId": "Dotseqn", + "detailsUrl": "https://spdx.org/licenses/MS-LPL.json", + "referenceNumber": 406, + "name": "Microsoft Limited Public License", + "licenseId": "MS-LPL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Dotseqn" + "https://www.openhub.net/licenses/mslpl", + "https://github.com/gabegundy/atlserver/blob/master/License.txt", + "https://en.wikipedia.org/wiki/Shared_Source_Initiative#Microsoft_Limited_Public_License_(Ms-LPL)" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SMLNJ.html", + "reference": "https://spdx.org/licenses/MS-PL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SMLNJ.json", - "referenceNumber": 312, - "name": "Standard ML of New Jersey License", - "licenseId": "SMLNJ", + "detailsUrl": "https://spdx.org/licenses/MS-PL.json", + "referenceNumber": 509, + "name": "Microsoft Public License", + "licenseId": "MS-PL", "seeAlso": [ - "https://www.smlnj.org/license.html" + "http://www.microsoft.com/opensource/licenses.mspx", + "https://opensource.org/licenses/MS-PL" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.2.2.html", + "reference": "https://spdx.org/licenses/MS-RL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.2.json", - "referenceNumber": 313, - "name": "Open LDAP Public License 2.2.2", - "licenseId": "OLDAP-2.2.2", - "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003ddf2cc1e21eb7c160695f5b7cffd6296c151ba188" - ], - "isOsiApproved": false - }, - { - "reference": "https://spdx.org/licenses/GPL-2.0-with-font-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-font-exception.json", - "referenceNumber": 314, - "name": "GNU General Public License v2.0 w/Font exception", - "licenseId": "GPL-2.0-with-font-exception", + "detailsUrl": "https://spdx.org/licenses/MS-RL.json", + "referenceNumber": 18, + "name": "Microsoft Reciprocal License", + "licenseId": "MS-RL", "seeAlso": [ - "https://www.gnu.org/licenses/gpl-faq.html#FontException" + "http://www.microsoft.com/opensource/licenses.mspx", + "https://opensource.org/licenses/MS-RL" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-3.0.html", + "reference": "https://spdx.org/licenses/MTLL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-3.0.json", - "referenceNumber": 315, - "name": "Creative Commons Attribution 3.0 Unported", - "licenseId": "CC-BY-3.0", + "detailsUrl": "https://spdx.org/licenses/MTLL.json", + "referenceNumber": 229, + "name": "Matrix Template Library License", + "licenseId": "MTLL", "seeAlso": [ - "https://creativecommons.org/licenses/by/3.0/legalcode" + "https://fedoraproject.org/wiki/Licensing/Matrix_Template_Library_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.html", + "reference": "https://spdx.org/licenses/MulanPSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.json", - "referenceNumber": 316, - "name": "PolyForm Noncommercial License 1.0.0", - "licenseId": "PolyForm-Noncommercial-1.0.0", + "detailsUrl": "https://spdx.org/licenses/MulanPSL-1.0.json", + "referenceNumber": 312, + "name": "Mulan Permissive Software License, Version 1", + "licenseId": "MulanPSL-1.0", "seeAlso": [ - "https://polyformproject.org/licenses/noncommercial/1.0.0" + "https://license.coscl.org.cn/MulanPSL/", + "https://github.com/yuwenlong/longphp/blob/25dfb70cc2a466dc4bb55ba30901cbce08d164b5/LICENSE" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MIT.html", + "reference": "https://spdx.org/licenses/MulanPSL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT.json", - "referenceNumber": 317, - "name": "MIT License", - "licenseId": "MIT", + "detailsUrl": "https://spdx.org/licenses/MulanPSL-2.0.json", + "referenceNumber": 74, + "name": "Mulan Permissive Software License, Version 2", + "licenseId": "MulanPSL-2.0", "seeAlso": [ - "https://opensource.org/licenses/MIT" + "https://license.coscl.org.cn/MulanPSL2/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CNRI-Python.html", + "reference": "https://spdx.org/licenses/Multics.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CNRI-Python.json", - "referenceNumber": 318, - "name": "CNRI Python License", - "licenseId": "CNRI-Python", + "detailsUrl": "https://spdx.org/licenses/Multics.json", + "referenceNumber": 236, + "name": "Multics License", + "licenseId": "Multics", "seeAlso": [ - "https://opensource.org/licenses/CNRI-Python" + "https://opensource.org/licenses/Multics" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/WTFPL.html", + "reference": "https://spdx.org/licenses/Mup.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/WTFPL.json", - "referenceNumber": 319, - "name": "Do What The F*ck You Want To Public License", - "licenseId": "WTFPL", + "detailsUrl": "https://spdx.org/licenses/Mup.json", + "referenceNumber": 98, + "name": "Mup License", + "licenseId": "Mup", "seeAlso": [ - "http://www.wtfpl.net/about/", - "http://sam.zoy.org/wtfpl/COPYING" + "https://fedoraproject.org/wiki/Licensing/Mup" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AFL-3.0.html", + "reference": "https://spdx.org/licenses/NAIST-2003.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-3.0.json", - "referenceNumber": 320, - "name": "Academic Free License v3.0", - "licenseId": "AFL-3.0", + "detailsUrl": "https://spdx.org/licenses/NAIST-2003.json", + "referenceNumber": 408, + "name": "Nara Institute of Science and Technology License (2003)", + "licenseId": "NAIST-2003", "seeAlso": [ - "http://www.rosenlaw.com/AFL3.0.htm", - "https://opensource.org/licenses/afl-3.0" + "https://enterprise.dejacode.com/licenses/public/naist-2003/#license-text", + "https://github.com/nodejs/node/blob/4a19cc8947b1bba2b2d27816ec3d0edf9b28e503/LICENSE#L343" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/bzip2-1.0.5.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/bzip2-1.0.5.json", - "referenceNumber": 321, - "name": "bzip2 and libbzip2 License v1.0.5", - "licenseId": "bzip2-1.0.5", + "reference": "https://spdx.org/licenses/NASA-1.3.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NASA-1.3.json", + "referenceNumber": 139, + "name": "NASA Open Source Agreement 1.3", + "licenseId": "NASA-1.3", "seeAlso": [ - "https://sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html", - "http://bzip.org/1.0.5/bzip2-manual-1.0.5.html" + "http://ti.arc.nasa.gov/opensource/nosa/", + "https://opensource.org/licenses/NASA-1.3" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.html", + "reference": "https://spdx.org/licenses/Naumen.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-1.0.json", - "referenceNumber": 322, - "name": "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", - "licenseId": "CC-BY-NC-SA-1.0", + "detailsUrl": "https://spdx.org/licenses/Naumen.json", + "referenceNumber": 371, + "name": "Naumen Public License", + "licenseId": "Naumen", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/1.0/legalcode" + "https://opensource.org/licenses/Naumen" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/SHL-0.5.html", + "reference": "https://spdx.org/licenses/NBPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SHL-0.5.json", - "referenceNumber": 323, - "name": "Solderpad Hardware License v0.5", - "licenseId": "SHL-0.5", + "detailsUrl": "https://spdx.org/licenses/NBPL-1.0.json", + "referenceNumber": 283, + "name": "Net Boolean Public License v1", + "licenseId": "NBPL-1.0", "seeAlso": [ - "https://solderpad.org/licenses/SHL-0.5/" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d37b4b3f6cc4bf34e1d3dec61e69914b9819d8894" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GD.html", + "reference": "https://spdx.org/licenses/NCGL-UK-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GD.json", - "referenceNumber": 324, - "name": "GD License", - "licenseId": "GD", + "detailsUrl": "https://spdx.org/licenses/NCGL-UK-2.0.json", + "referenceNumber": 49, + "name": "Non-Commercial Government Licence", + "licenseId": "NCGL-UK-2.0", "seeAlso": [ - "https://libgd.github.io/manuals/2.3.0/files/license-txt.html" + "http://www.nationalarchives.gov.uk/doc/non-commercial-government-licence/version/2/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.html", + "reference": "https://spdx.org/licenses/NCSA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.5.json", - "referenceNumber": 325, - "name": "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", - "licenseId": "CC-BY-NC-ND-2.5", + "detailsUrl": "https://spdx.org/licenses/NCSA.json", + "referenceNumber": 207, + "name": "University of Illinois/NCSA Open Source License", + "licenseId": "NCSA", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/2.5/legalcode" + "http://otm.illinois.edu/uiuc_openSource", + "https://opensource.org/licenses/NCSA" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NTP.html", + "reference": "https://spdx.org/licenses/Net-SNMP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NTP.json", - "referenceNumber": 326, - "name": "NTP License", - "licenseId": "NTP", + "detailsUrl": "https://spdx.org/licenses/Net-SNMP.json", + "referenceNumber": 347, + "name": "Net-SNMP License", + "licenseId": "Net-SNMP", "seeAlso": [ - "https://opensource.org/licenses/NTP" + "http://net-snmp.sourceforge.net/about/license.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CrystalStacker.html", + "reference": "https://spdx.org/licenses/NetCDF.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CrystalStacker.json", - "referenceNumber": 327, - "name": "CrystalStacker License", - "licenseId": "CrystalStacker", + "detailsUrl": "https://spdx.org/licenses/NetCDF.json", + "referenceNumber": 348, + "name": "NetCDF license", + "licenseId": "NetCDF", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:CrystalStacker?rd\u003dLicensing/CrystalStacker" + "http://www.unidata.ucar.edu/software/netcdf/copyright.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Baekmuk.html", + "reference": "https://spdx.org/licenses/Newsletr.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Baekmuk.json", - "referenceNumber": 328, - "name": "Baekmuk License", - "licenseId": "Baekmuk", + "detailsUrl": "https://spdx.org/licenses/Newsletr.json", + "referenceNumber": 221, + "name": "Newsletr License", + "licenseId": "Newsletr", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:Baekmuk?rd\u003dLicensing/Baekmuk" + "https://fedoraproject.org/wiki/Licensing/Newsletr" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AML.html", + "reference": "https://spdx.org/licenses/NGPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AML.json", - "referenceNumber": 329, - "name": "Apple MIT License", - "licenseId": "AML", + "detailsUrl": "https://spdx.org/licenses/NGPL.json", + "referenceNumber": 311, + "name": "Nethack General Public License", + "licenseId": "NGPL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Apple_MIT_License" + "https://opensource.org/licenses/NGPL" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/XSkat.html", + "reference": "https://spdx.org/licenses/NICTA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/XSkat.json", - "referenceNumber": 330, - "name": "XSkat License", - "licenseId": "XSkat", + "detailsUrl": "https://spdx.org/licenses/NICTA-1.0.json", + "referenceNumber": 107, + "name": "NICTA Public Software License, Version 1.0", + "licenseId": "NICTA-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/XSkat_License" + "https://opensource.apple.com/source/mDNSResponder/mDNSResponder-320.10/mDNSPosix/nss_ReadMe.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OGTSL.html", + "reference": "https://spdx.org/licenses/NIST-PD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGTSL.json", - "referenceNumber": 331, - "name": "Open Group Test Suite License", - "licenseId": "OGTSL", + "detailsUrl": "https://spdx.org/licenses/NIST-PD.json", + "referenceNumber": 182, + "name": "NIST Public Domain Notice", + "licenseId": "NIST-PD", "seeAlso": [ - "http://www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt", - "https://opensource.org/licenses/OGTSL" + "https://github.com/tcheneau/simpleRPL/blob/e645e69e38dd4e3ccfeceb2db8cba05b7c2e0cd3/LICENSE.txt", + "https://github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Motosoto.html", + "reference": "https://spdx.org/licenses/NIST-PD-fallback.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Motosoto.json", - "referenceNumber": 332, - "name": "Motosoto License", - "licenseId": "Motosoto", + "detailsUrl": "https://spdx.org/licenses/NIST-PD-fallback.json", + "referenceNumber": 357, + "name": "NIST Public Domain Notice with license fallback", + "licenseId": "NIST-PD-fallback", "seeAlso": [ - "https://opensource.org/licenses/Motosoto" + "https://github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE", + "https://github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/PHP-3.01.html", + "reference": "https://spdx.org/licenses/NLOD-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PHP-3.01.json", - "referenceNumber": 333, - "name": "PHP License v3.01", - "licenseId": "PHP-3.01", + "detailsUrl": "https://spdx.org/licenses/NLOD-1.0.json", + "referenceNumber": 496, + "name": "Norwegian Licence for Open Government Data (NLOD) 1.0", + "licenseId": "NLOD-1.0", "seeAlso": [ - "http://www.php.net/license/3_01.txt" + "http://data.norge.no/nlod/en/1.0" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/etalab-2.0.html", + "reference": "https://spdx.org/licenses/NLOD-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/etalab-2.0.json", - "referenceNumber": 334, - "name": "Etalab Open License 2.0", - "licenseId": "etalab-2.0", + "detailsUrl": "https://spdx.org/licenses/NLOD-2.0.json", + "referenceNumber": 341, + "name": "Norwegian Licence for Open Government Data (NLOD) 2.0", + "licenseId": "NLOD-2.0", "seeAlso": [ - "https://github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf", - "https://raw.githubusercontent.com/DISIC/politique-de-contribution-open-source/master/LICENSE" + "http://data.norge.no/nlod/en/2.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MPL-1.0.html", + "reference": "https://spdx.org/licenses/NLPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-1.0.json", - "referenceNumber": 335, - "name": "Mozilla Public License 1.0", - "licenseId": "MPL-1.0", + "detailsUrl": "https://spdx.org/licenses/NLPL.json", + "referenceNumber": 138, + "name": "No Limit Public License", + "licenseId": "NLPL", "seeAlso": [ - "http://www.mozilla.org/MPL/MPL-1.0.html", - "https://opensource.org/licenses/MPL-1.0" + "https://fedoraproject.org/wiki/Licensing/NLPL" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0+.json", - "referenceNumber": 336, - "name": "GNU General Public License v2.0 or later", - "licenseId": "GPL-2.0+", + "reference": "https://spdx.org/licenses/Nokia.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/Nokia.json", + "referenceNumber": 441, + "name": "Nokia Open Source License", + "licenseId": "Nokia", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "https://opensource.org/licenses/nokia" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/StandardML-NJ.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/StandardML-NJ.json", - "referenceNumber": 337, - "name": "Standard ML of New Jersey License", - "licenseId": "StandardML-NJ", + "reference": "https://spdx.org/licenses/NOSL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/NOSL.json", + "referenceNumber": 193, + "name": "Netizen Open Source License", + "licenseId": "NOSL", "seeAlso": [ - "http://www.smlnj.org//license.html" + "http://bits.netizen.com.au/licenses/NOSL/nosl.txt" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Interbase-1.0.html", + "reference": "https://spdx.org/licenses/Noweb.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Interbase-1.0.json", - "referenceNumber": 338, - "name": "Interbase Public License v1.0", - "licenseId": "Interbase-1.0", + "detailsUrl": "https://spdx.org/licenses/Noweb.json", + "referenceNumber": 335, + "name": "Noweb License", + "licenseId": "Noweb", "seeAlso": [ - "https://web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html" + "https://fedoraproject.org/wiki/Licensing/Noweb" ], "isOsiApproved": false }, @@ -4268,7 +4561,7 @@ "reference": "https://spdx.org/licenses/NPL-1.0.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/NPL-1.0.json", - "referenceNumber": 339, + "referenceNumber": 328, "name": "Netscape Public License v1.0", "licenseId": "NPL-1.0", "seeAlso": [ @@ -4278,233 +4571,199 @@ "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/eGenix.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/eGenix.json", - "referenceNumber": 340, - "name": "eGenix.com Public License 1.1.0", - "licenseId": "eGenix", - "seeAlso": [ - "http://www.egenix.com/products/eGenix.com-Public-License-1.1.0.pdf", - "https://fedoraproject.org/wiki/Licensing/eGenix.com_Public_License_1.1.0" - ], - "isOsiApproved": false - }, - { - "reference": "https://spdx.org/licenses/NICTA-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NICTA-1.0.json", - "referenceNumber": 341, - "name": "NICTA Public Software License, Version 1.0", - "licenseId": "NICTA-1.0", - "seeAlso": [ - "https://opensource.apple.com/source/mDNSResponder/mDNSResponder-320.10/mDNSPosix/nss_ReadMe.txt" - ], - "isOsiApproved": false - }, - { - "reference": "https://spdx.org/licenses/PSF-2.0.html", + "reference": "https://spdx.org/licenses/NPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PSF-2.0.json", - "referenceNumber": 342, - "name": "Python Software Foundation License 2.0", - "licenseId": "PSF-2.0", + "detailsUrl": "https://spdx.org/licenses/NPL-1.1.json", + "referenceNumber": 52, + "name": "Netscape Public License v1.1", + "licenseId": "NPL-1.1", "seeAlso": [ - "https://opensource.org/licenses/Python-2.0" + "http://www.mozilla.org/MPL/NPL/1.1/" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-invariants-only.html", + "reference": "https://spdx.org/licenses/NPOSL-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-invariants-only.json", - "referenceNumber": 343, - "name": "GNU Free Documentation License v1.3 only - invariants", - "licenseId": "GFDL-1.3-invariants-only", + "detailsUrl": "https://spdx.org/licenses/NPOSL-3.0.json", + "referenceNumber": 286, + "name": "Non-Profit Open Software License 3.0", + "licenseId": "NPOSL-3.0", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "https://opensource.org/licenses/NOSL3.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OGL-UK-3.0.html", + "reference": "https://spdx.org/licenses/NRL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-UK-3.0.json", - "referenceNumber": 344, - "name": "Open Government Licence v3.0", - "licenseId": "OGL-UK-3.0", + "detailsUrl": "https://spdx.org/licenses/NRL.json", + "referenceNumber": 454, + "name": "NRL License", + "licenseId": "NRL", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" + "http://web.mit.edu/network/isakmp/nrllicense.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/dvipdfm.html", + "reference": "https://spdx.org/licenses/NTP.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/dvipdfm.json", - "referenceNumber": 345, - "name": "dvipdfm License", - "licenseId": "dvipdfm", + "detailsUrl": "https://spdx.org/licenses/NTP.json", + "referenceNumber": 255, + "name": "NTP License", + "licenseId": "NTP", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/dvipdfm" + "https://opensource.org/licenses/NTP" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/DRL-1.0.html", + "reference": "https://spdx.org/licenses/NTP-0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/DRL-1.0.json", - "referenceNumber": 346, - "name": "Detection Rule License 1.0", - "licenseId": "DRL-1.0", + "detailsUrl": "https://spdx.org/licenses/NTP-0.json", + "referenceNumber": 379, + "name": "NTP No Attribution", + "licenseId": "NTP-0", "seeAlso": [ - "https://github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md" + "https://github.com/tytso/e2fsprogs/blob/master/lib/et/et_name.c" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Ruby.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Ruby.json", - "referenceNumber": 347, - "name": "Ruby License", - "licenseId": "Ruby", + "reference": "https://spdx.org/licenses/Nunit.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/Nunit.json", + "referenceNumber": 244, + "name": "Nunit License", + "licenseId": "Nunit", "seeAlso": [ - "http://www.ruby-lang.org/en/LICENSE.txt" + "https://fedoraproject.org/wiki/Licensing/Nunit" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.html", + "reference": "https://spdx.org/licenses/O-UDA-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json", - "referenceNumber": 348, - "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", - "licenseId": "CC-BY-NC-ND-3.0-IGO", + "detailsUrl": "https://spdx.org/licenses/O-UDA-1.0.json", + "referenceNumber": 4, + "name": "Open Use of Data Agreement v1.0", + "licenseId": "O-UDA-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/3.0/igo/legalcode" + "https://github.com/microsoft/Open-Use-of-Data-Agreement/blob/v1.0/O-UDA-1.0.md", + "https://cdla.dev/open-use-of-data-agreement-v1-0/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/APAFML.html", + "reference": "https://spdx.org/licenses/OCCT-PL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APAFML.json", - "referenceNumber": 349, - "name": "Adobe Postscript AFM License", - "licenseId": "APAFML", + "detailsUrl": "https://spdx.org/licenses/OCCT-PL.json", + "referenceNumber": 112, + "name": "Open CASCADE Technology Public License", + "licenseId": "OCCT-PL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AdobePostscriptAFM" + "http://www.opencascade.com/content/occt-public-license" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CDLA-Permissive-2.0.html", + "reference": "https://spdx.org/licenses/OCLC-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CDLA-Permissive-2.0.json", - "referenceNumber": 350, - "name": "Community Data License Agreement Permissive 2.0", - "licenseId": "CDLA-Permissive-2.0", + "detailsUrl": "https://spdx.org/licenses/OCLC-2.0.json", + "referenceNumber": 222, + "name": "OCLC Research Public License 2.0", + "licenseId": "OCLC-2.0", "seeAlso": [ - "https://cdla.dev/permissive-2-0" + "http://www.oclc.org/research/activities/software/license/v2final.htm", + "https://opensource.org/licenses/OCLC-2.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-2.0.html", + "reference": "https://spdx.org/licenses/ODbL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-2.0.json", - "referenceNumber": 351, - "name": "Creative Commons Attribution Non Commercial 2.0 Generic", - "licenseId": "CC-BY-NC-2.0", + "detailsUrl": "https://spdx.org/licenses/ODbL-1.0.json", + "referenceNumber": 365, + "name": "Open Data Commons Open Database License v1.0", + "licenseId": "ODbL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/2.0/legalcode" + "http://www.opendatacommons.org/licenses/odbl/1.0/", + "https://opendatacommons.org/licenses/odbl/1-0/" ], "isOsiApproved": false, - "isFsfLibre": false + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.html", + "reference": "https://spdx.org/licenses/ODC-By-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-3.0-DE.json", - "referenceNumber": 352, - "name": "Creative Commons Attribution No Derivatives 3.0 Germany", - "licenseId": "CC-BY-ND-3.0-DE", + "detailsUrl": "https://spdx.org/licenses/ODC-By-1.0.json", + "referenceNumber": 300, + "name": "Open Data Commons Attribution License v1.0", + "licenseId": "ODC-By-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/3.0/de/legalcode" + "https://opendatacommons.org/licenses/by/1.0/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OFL-1.1-RFN.html", + "reference": "https://spdx.org/licenses/OFL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.1-RFN.json", - "referenceNumber": 353, - "name": "SIL Open Font License 1.1 with Reserved Font Name", - "licenseId": "OFL-1.1-RFN", + "detailsUrl": "https://spdx.org/licenses/OFL-1.0.json", + "referenceNumber": 288, + "name": "SIL Open Font License 1.0", + "licenseId": "OFL-1.0", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", - "https://opensource.org/licenses/OFL-1.1" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Caldera.html", + "reference": "https://spdx.org/licenses/OFL-1.0-no-RFN.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Caldera.json", - "referenceNumber": 354, - "name": "Caldera License", - "licenseId": "Caldera", + "detailsUrl": "https://spdx.org/licenses/OFL-1.0-no-RFN.json", + "referenceNumber": 160, + "name": "SIL Open Font License 1.0 with no Reserved Font Name", + "licenseId": "OFL-1.0-no-RFN", "seeAlso": [ - "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-3.0-with-GCC-exception.json", - "referenceNumber": 355, - "name": "GNU General Public License v3.0 w/GCC Runtime Library exception", - "licenseId": "GPL-3.0-with-GCC-exception", + "reference": "https://spdx.org/licenses/OFL-1.0-RFN.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OFL-1.0-RFN.json", + "referenceNumber": 151, + "name": "SIL Open Font License 1.0 with Reserved Font Name", + "licenseId": "OFL-1.0-RFN", "seeAlso": [ - "https://www.gnu.org/licenses/gcc-exception-3.1.html" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPL-1.02.html", + "reference": "https://spdx.org/licenses/OFL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPL-1.02.json", - "referenceNumber": 356, - "name": "Lucent Public License v1.02", - "licenseId": "LPL-1.02", + "detailsUrl": "https://spdx.org/licenses/OFL-1.1.json", + "referenceNumber": 85, + "name": "SIL Open Font License 1.1", + "licenseId": "OFL-1.1", "seeAlso": [ - "http://plan9.bell-labs.com/plan9/license.html", - "https://opensource.org/licenses/LPL-1.02" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", + "https://opensource.org/licenses/OFL-1.1" ], "isOsiApproved": true, "isFsfLibre": true }, - { - "reference": "https://spdx.org/licenses/Beerware.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Beerware.json", - "referenceNumber": 357, - "name": "Beerware License", - "licenseId": "Beerware", - "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Beerware", - "https://people.freebsd.org/~phk/" - ], - "isOsiApproved": false - }, { "reference": "https://spdx.org/licenses/OFL-1.1-no-RFN.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/OFL-1.1-no-RFN.json", - "referenceNumber": 358, + "referenceNumber": 477, "name": "SIL Open Font License 1.1 with no Reserved Font Name", "licenseId": "OFL-1.1-no-RFN", "seeAlso": [ @@ -4514,484 +4773,464 @@ "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OLDAP-1.3.html", + "reference": "https://spdx.org/licenses/OFL-1.1-RFN.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-1.3.json", - "referenceNumber": 359, - "name": "Open LDAP Public License v1.3", - "licenseId": "OLDAP-1.3", + "detailsUrl": "https://spdx.org/licenses/OFL-1.1-RFN.json", + "referenceNumber": 114, + "name": "SIL Open Font License 1.1 with Reserved Font Name", + "licenseId": "OFL-1.1-RFN", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003de5f8117f0ce088d0bd7a8e18ddf37eaa40eb09b1" + "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL_web", + "https://opensource.org/licenses/OFL-1.1" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/APL-1.0.html", + "reference": "https://spdx.org/licenses/OGC-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APL-1.0.json", - "referenceNumber": 360, - "name": "Adaptive Public License 1.0", - "licenseId": "APL-1.0", + "detailsUrl": "https://spdx.org/licenses/OGC-1.0.json", + "referenceNumber": 410, + "name": "OGC Software License, Version 1.0", + "licenseId": "OGC-1.0", "seeAlso": [ - "https://opensource.org/licenses/APL-1.0" + "https://www.ogc.org/ogc/software/1.0" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Linux-man-pages-copyleft.html", + "reference": "https://spdx.org/licenses/OGDL-Taiwan-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Linux-man-pages-copyleft.json", - "referenceNumber": 361, - "name": "Linux man-pages Copyleft", - "licenseId": "Linux-man-pages-copyleft", + "detailsUrl": "https://spdx.org/licenses/OGDL-Taiwan-1.0.json", + "referenceNumber": 110, + "name": "Taiwan Open Government Data License, version 1.0", + "licenseId": "OGDL-Taiwan-1.0", "seeAlso": [ - "https://www.kernel.org/doc/man-pages/licenses.html" + "https://data.gov.tw/license" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/curl.html", + "reference": "https://spdx.org/licenses/OGL-Canada-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/curl.json", - "referenceNumber": 362, - "name": "curl License", - "licenseId": "curl", + "detailsUrl": "https://spdx.org/licenses/OGL-Canada-2.0.json", + "referenceNumber": 99, + "name": "Open Government Licence - Canada", + "licenseId": "OGL-Canada-2.0", "seeAlso": [ - "https://github.com/bagder/curl/blob/master/COPYING" + "https://open.canada.ca/en/open-government-licence-canada" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-or-later.html", + "reference": "https://spdx.org/licenses/OGL-UK-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-or-later.json", - "referenceNumber": 363, - "name": "GNU General Public License v2.0 or later", - "licenseId": "GPL-2.0-or-later", + "detailsUrl": "https://spdx.org/licenses/OGL-UK-1.0.json", + "referenceNumber": 202, + "name": "Open Government Licence v1.0", + "licenseId": "OGL-UK-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html", - "https://opensource.org/licenses/GPL-2.0" + "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/1/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SISSL.html", + "reference": "https://spdx.org/licenses/OGL-UK-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SISSL.json", - "referenceNumber": 364, - "name": "Sun Industry Standards Source License v1.1", - "licenseId": "SISSL", + "detailsUrl": "https://spdx.org/licenses/OGL-UK-2.0.json", + "referenceNumber": 381, + "name": "Open Government Licence v2.0", + "licenseId": "OGL-UK-2.0", "seeAlso": [ - "http://www.openoffice.org/licenses/sissl_license.html", - "https://opensource.org/licenses/SISSL" + "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-4.0.html", + "reference": "https://spdx.org/licenses/OGL-UK-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-4.0.json", - "referenceNumber": 365, - "name": "Creative Commons Attribution 4.0 International", - "licenseId": "CC-BY-4.0", + "detailsUrl": "https://spdx.org/licenses/OGL-UK-3.0.json", + "referenceNumber": 187, + "name": "Open Government Licence v3.0", + "licenseId": "OGL-UK-3.0", "seeAlso": [ - "https://creativecommons.org/licenses/by/4.0/legalcode" + "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CECILL-1.1.html", + "reference": "https://spdx.org/licenses/OGTSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-1.1.json", - "referenceNumber": 366, - "name": "CeCILL Free Software License Agreement v1.1", - "licenseId": "CECILL-1.1", + "detailsUrl": "https://spdx.org/licenses/OGTSL.json", + "referenceNumber": 89, + "name": "Open Group Test Suite License", + "licenseId": "OGTSL", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V1.1-US.html" + "http://www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt", + "https://opensource.org/licenses/OGTSL" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/OGL-UK-1.0.html", + "reference": "https://spdx.org/licenses/OLDAP-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OGL-UK-1.0.json", - "referenceNumber": 367, - "name": "Open Government Licence v1.0", - "licenseId": "OGL-UK-1.0", + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.1.json", + "referenceNumber": 486, + "name": "Open LDAP Public License v1.1", + "licenseId": "OLDAP-1.1", "seeAlso": [ - "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/1/" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d806557a5ad59804ef3a44d5abfbe91d706b0791f" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-bison-exception.json", - "referenceNumber": 368, - "name": "GNU General Public License v2.0 w/Bison exception", - "licenseId": "GPL-2.0-with-bison-exception", + "reference": "https://spdx.org/licenses/OLDAP-1.2.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.2.json", + "referenceNumber": 377, + "name": "Open LDAP Public License v1.2", + "licenseId": "OLDAP-1.2", "seeAlso": [ - "http://git.savannah.gnu.org/cgit/bison.git/tree/data/yacc.c?id\u003d193d7c7054ba7197b0789e14965b739162319b5e#n141" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d42b0383c50c299977b5893ee695cf4e486fb0dc7" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OLDAP-2.7.html", + "reference": "https://spdx.org/licenses/OLDAP-1.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.7.json", - "referenceNumber": 369, - "name": "Open LDAP Public License v2.7", - "licenseId": "OLDAP-2.7", + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.3.json", + "referenceNumber": 1, + "name": "Open LDAP Public License v1.3", + "licenseId": "OLDAP-1.3", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d47c2415c1df81556eeb39be6cad458ef87c534a2" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003de5f8117f0ce088d0bd7a8e18ddf37eaa40eb09b1" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Glulxe.html", + "reference": "https://spdx.org/licenses/OLDAP-1.4.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Glulxe.json", - "referenceNumber": 370, - "name": "Glulxe License", - "licenseId": "Glulxe", + "detailsUrl": "https://spdx.org/licenses/OLDAP-1.4.json", + "referenceNumber": 445, + "name": "Open LDAP Public License v1.4", + "licenseId": "OLDAP-1.4", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Glulxe" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dc9f95c2f3f2ffb5e0ae55fe7388af75547660941" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-4-Clause-UC.html", + "reference": "https://spdx.org/licenses/OLDAP-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause-UC.json", - "referenceNumber": 371, - "name": "BSD-4-Clause (University of California-Specific)", - "licenseId": "BSD-4-Clause-UC", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.json", + "referenceNumber": 155, + "name": "Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B)", + "licenseId": "OLDAP-2.0", "seeAlso": [ - "http://www.freebsd.org/copyright/license.html" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcbf50f4e1185a21abd4c0a54d3f4341fe28f36ea" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LPPL-1.3a.html", + "reference": "https://spdx.org/licenses/OLDAP-2.0.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.3a.json", - "referenceNumber": 372, - "name": "LaTeX Project Public License v1.3a", - "licenseId": "LPPL-1.3a", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.0.1.json", + "referenceNumber": 169, + "name": "Open LDAP Public License v2.0.1", + "licenseId": "OLDAP-2.0.1", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-3a.txt" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db6d68acd14e51ca3aab4428bf26522aa74873f0e" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Zlib.html", + "reference": "https://spdx.org/licenses/OLDAP-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zlib.json", - "referenceNumber": 373, - "name": "zlib License", - "licenseId": "Zlib", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.1.json", + "referenceNumber": 248, + "name": "Open LDAP Public License v2.1", + "licenseId": "OLDAP-2.1", "seeAlso": [ - "http://www.zlib.net/zlib_license.html", - "https://opensource.org/licenses/Zlib" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003db0d176738e96a0d3b9f85cb51e140a86f21be715" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Crossword.html", + "reference": "https://spdx.org/licenses/OLDAP-2.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Crossword.json", - "referenceNumber": 374, - "name": "Crossword License", - "licenseId": "Crossword", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.json", + "referenceNumber": 499, + "name": "Open LDAP Public License v2.2", + "licenseId": "OLDAP-2.2", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Crossword" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d470b0c18ec67621c85881b2733057fecf4a1acc3" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GFDL-1.1.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1.json", - "referenceNumber": 375, - "name": "GNU Free Documentation License v1.1", - "licenseId": "GFDL-1.1", + "reference": "https://spdx.org/licenses/OLDAP-2.2.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.1.json", + "referenceNumber": 2, + "name": "Open LDAP Public License v2.2.1", + "licenseId": "OLDAP-2.2.1", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d4bc786f34b50aa301be6f5600f58a980070f481e" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-4-Clause.html", + "reference": "https://spdx.org/licenses/OLDAP-2.2.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-4-Clause.json", - "referenceNumber": 376, - "name": "BSD 4-Clause \"Original\" or \"Old\" License", - "licenseId": "BSD-4-Clause", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.2.2.json", + "referenceNumber": 338, + "name": "Open LDAP Public License 2.2.2", + "licenseId": "OLDAP-2.2.2", "seeAlso": [ - "http://directory.fsf.org/wiki/License:BSD_4Clause" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003ddf2cc1e21eb7c160695f5b7cffd6296c151ba188" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Condor-1.1.html", + "reference": "https://spdx.org/licenses/OLDAP-2.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Condor-1.1.json", - "referenceNumber": 377, - "name": "Condor Public License v1.1", - "licenseId": "Condor-1.1", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.3.json", + "referenceNumber": 103, + "name": "Open LDAP Public License v2.3", + "licenseId": "OLDAP-2.3", "seeAlso": [ - "http://research.cs.wisc.edu/condor/license.html#condor", - "http://web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dd32cf54a32d581ab475d23c810b0a7fbaf8d63c3" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/SSH-short.html", + "reference": "https://spdx.org/licenses/OLDAP-2.4.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SSH-short.json", - "referenceNumber": 378, - "name": "SSH short notice", - "licenseId": "SSH-short", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.4.json", + "referenceNumber": 382, + "name": "Open LDAP Public License v2.4", + "licenseId": "OLDAP-2.4", "seeAlso": [ - "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h", - "http://web.mit.edu/kolya/.f/root/athena.mit.edu/sipb.mit.edu/project/openssh/OldFiles/src/openssh-2.9.9p2/ssh-add.1", - "https://joinup.ec.europa.eu/svn/lesoll/trunk/italc/lib/src/dsa_key.cpp" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003dcd1284c4a91a8a380d904eee68d1583f989ed386" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Borceux.html", + "reference": "https://spdx.org/licenses/OLDAP-2.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Borceux.json", - "referenceNumber": 379, - "name": "Borceux license", - "licenseId": "Borceux", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.5.json", + "referenceNumber": 466, + "name": "Open LDAP Public License v2.5", + "licenseId": "OLDAP-2.5", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Borceux" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d6852b9d90022e8593c98205413380536b1b5a7cf" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/GPL-1.0-only.html", + "reference": "https://spdx.org/licenses/OLDAP-2.6.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0-only.json", - "referenceNumber": 380, - "name": "GNU General Public License v1.0 only", - "licenseId": "GPL-1.0-only", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.6.json", + "referenceNumber": 384, + "name": "Open LDAP Public License v2.6", + "licenseId": "OLDAP-2.6", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d1cae062821881f41b73012ba816434897abf4205" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Clear.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Clear.json", - "referenceNumber": 381, - "name": "BSD 3-Clause Clear License", - "licenseId": "BSD-3-Clause-Clear", - "seeAlso": [ - "http://labs.metacarta.com/license-explanation.html#license" - ], - "isOsiApproved": false, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/FTL.html", + "reference": "https://spdx.org/licenses/OLDAP-2.7.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FTL.json", - "referenceNumber": 382, - "name": "Freetype Project License", - "licenseId": "FTL", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.7.json", + "referenceNumber": 414, + "name": "Open LDAP Public License v2.7", + "licenseId": "OLDAP-2.7", "seeAlso": [ - "http://freetype.fis.uniroma2.it/FTL.TXT", - "http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT", - "http://gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT" + "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d47c2415c1df81556eeb39be6cad458ef87c534a2" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.html", + "reference": "https://spdx.org/licenses/OLDAP-2.8.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-3.0-AT.json", - "referenceNumber": 383, - "name": "Creative Commons Attribution Share Alike 3.0 Austria", - "licenseId": "CC-BY-SA-3.0-AT", + "detailsUrl": "https://spdx.org/licenses/OLDAP-2.8.json", + "referenceNumber": 354, + "name": "Open LDAP Public License v2.8", + "licenseId": "OLDAP-2.8", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/3.0/at/legalcode" + "http://www.openldap.org/software/release/license.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Hippocratic-2.1.html", + "reference": "https://spdx.org/licenses/OML.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Hippocratic-2.1.json", - "referenceNumber": 384, - "name": "Hippocratic License 2.1", - "licenseId": "Hippocratic-2.1", + "detailsUrl": "https://spdx.org/licenses/OML.json", + "referenceNumber": 464, + "name": "Open Market License", + "licenseId": "OML", "seeAlso": [ - "https://firstdonoharm.dev/version/2/1/license.html", - "https://github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt" + "https://fedoraproject.org/wiki/Licensing/Open_Market_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Spencer-99.html", + "reference": "https://spdx.org/licenses/OpenSSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Spencer-99.json", - "referenceNumber": 385, - "name": "Spencer License 99", - "licenseId": "Spencer-99", + "detailsUrl": "https://spdx.org/licenses/OpenSSL.json", + "referenceNumber": 402, + "name": "OpenSSL License", + "licenseId": "OpenSSL", "seeAlso": [ - "http://www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c" + "http://www.openssl.org/source/license.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-only.html", + "reference": "https://spdx.org/licenses/OPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-only.json", - "referenceNumber": 386, - "name": "GNU Free Documentation License v1.2 only", - "licenseId": "GFDL-1.2-only", + "detailsUrl": "https://spdx.org/licenses/OPL-1.0.json", + "referenceNumber": 142, + "name": "Open Public License v1.0", + "licenseId": "OPL-1.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "http://old.koalateam.com/jackaroo/OPL_1_0.TXT", + "https://fedoraproject.org/wiki/Licensing/Open_Public_License" ], "isOsiApproved": false, - "isFsfLibre": true + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/Intel-ACPI.html", + "reference": "https://spdx.org/licenses/OPUBL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Intel-ACPI.json", - "referenceNumber": 387, - "name": "Intel ACPI Software License Agreement", - "licenseId": "Intel-ACPI", + "detailsUrl": "https://spdx.org/licenses/OPUBL-1.0.json", + "referenceNumber": 375, + "name": "Open Publication License v1.0", + "licenseId": "OPUBL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Intel_ACPI_Software_License_Agreement" + "http://opencontent.org/openpub/", + "https://www.debian.org/opl", + "https://www.ctan.org/license/opl" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/AFL-2.0.html", + "reference": "https://spdx.org/licenses/OSET-PL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/AFL-2.0.json", - "referenceNumber": 388, - "name": "Academic Free License v2.0", - "licenseId": "AFL-2.0", + "detailsUrl": "https://spdx.org/licenses/OSET-PL-2.1.json", + "referenceNumber": 50, + "name": "OSET Public License version 2.1", + "licenseId": "OSET-PL-2.1", "seeAlso": [ - "http://wayback.archive.org/web/20060924134533/http://www.opensource.org/licenses/afl-2.0.txt" + "http://www.osetfoundation.org/public-license", + "https://opensource.org/licenses/OPL-2.1" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.html", + "reference": "https://spdx.org/licenses/OSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Military-License.json", - "referenceNumber": 389, - "name": "BSD 3-Clause No Military License", - "licenseId": "BSD-3-Clause-No-Military-License", + "detailsUrl": "https://spdx.org/licenses/OSL-1.0.json", + "referenceNumber": 403, + "name": "Open Software License 1.0", + "licenseId": "OSL-1.0", "seeAlso": [ - "https://gitlab.syncad.com/hive/dhive/-/blob/master/LICENSE", - "https://github.com/greymass/swift-eosio/blob/master/LICENSE" + "https://opensource.org/licenses/OSL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Libpng.html", + "reference": "https://spdx.org/licenses/OSL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Libpng.json", - "referenceNumber": 390, - "name": "libpng License", - "licenseId": "Libpng", + "detailsUrl": "https://spdx.org/licenses/OSL-1.1.json", + "referenceNumber": 400, + "name": "Open Software License 1.1", + "licenseId": "OSL-1.1", "seeAlso": [ - "http://www.libpng.org/pub/png/src/libpng-LICENSE.txt" + "https://fedoraproject.org/wiki/Licensing/OSL1.1" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OFL-1.0.html", + "reference": "https://spdx.org/licenses/OSL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.0.json", - "referenceNumber": 391, - "name": "SIL Open Font License 1.0", - "licenseId": "OFL-1.0", + "detailsUrl": "https://spdx.org/licenses/OSL-2.0.json", + "referenceNumber": 364, + "name": "Open Software License 2.0", + "licenseId": "OSL-2.0", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" + "http://web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MTLL.html", + "reference": "https://spdx.org/licenses/OSL-2.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MTLL.json", - "referenceNumber": 392, - "name": "Matrix Template Library License", - "licenseId": "MTLL", + "detailsUrl": "https://spdx.org/licenses/OSL-2.1.json", + "referenceNumber": 159, + "name": "Open Software License 2.1", + "licenseId": "OSL-2.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Matrix_Template_Library_License" + "http://web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm", + "https://opensource.org/licenses/OSL-2.1" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.html", + "reference": "https://spdx.org/licenses/OSL-3.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json", - "referenceNumber": 393, - "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", - "licenseId": "CC-BY-NC-ND-3.0-DE", + "detailsUrl": "https://spdx.org/licenses/OSL-3.0.json", + "referenceNumber": 342, + "name": "Open Software License 3.0", + "licenseId": "OSL-3.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/3.0/de/legalcode" + "https://web.archive.org/web/20120101081418/http://rosenlaw.com:80/OSL3.0.htm", + "https://opensource.org/licenses/OSL-3.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/EFL-1.0.html", + "reference": "https://spdx.org/licenses/Parity-6.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/EFL-1.0.json", - "referenceNumber": 394, - "name": "Eiffel Forum License v1.0", - "licenseId": "EFL-1.0", + "detailsUrl": "https://spdx.org/licenses/Parity-6.0.0.json", + "referenceNumber": 102, + "name": "The Parity Public License 6.0.0", + "licenseId": "Parity-6.0.0", "seeAlso": [ - "http://www.eiffel-nice.org/license/forum.txt", - "https://opensource.org/licenses/EFL-1.0" + "https://paritylicense.com/versions/6.0.0.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FDK-AAC.html", + "reference": "https://spdx.org/licenses/Parity-7.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FDK-AAC.json", - "referenceNumber": 395, - "name": "Fraunhofer FDK AAC Codec Library", - "licenseId": "FDK-AAC", + "detailsUrl": "https://spdx.org/licenses/Parity-7.0.0.json", + "referenceNumber": 51, + "name": "The Parity Public License 7.0.0", + "licenseId": "Parity-7.0.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/FDK-AAC", - "https://directory.fsf.org/wiki/License:Fdk" + "https://paritylicense.com/versions/7.0.0.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CECILL-1.0.html", + "reference": "https://spdx.org/licenses/PDDL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-1.0.json", - "referenceNumber": 396, - "name": "CeCILL Free Software License Agreement v1.0", - "licenseId": "CECILL-1.0", + "detailsUrl": "https://spdx.org/licenses/PDDL-1.0.json", + "referenceNumber": 87, + "name": "Open Data Commons Public Domain Dedication \u0026 License 1.0", + "licenseId": "PDDL-1.0", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V1-fr.html" + "http://opendatacommons.org/licenses/pddl/1.0/", + "https://opendatacommons.org/licenses/pddl/" ], "isOsiApproved": false }, @@ -4999,7 +5238,7 @@ "reference": "https://spdx.org/licenses/PHP-3.0.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/PHP-3.0.json", - "referenceNumber": 397, + "referenceNumber": 191, "name": "PHP License v3.0", "licenseId": "PHP-3.0", "seeAlso": [ @@ -5009,887 +5248,829 @@ "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/FreeImage.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FreeImage.json", - "referenceNumber": 398, - "name": "FreeImage Public License v1.0", - "licenseId": "FreeImage", - "seeAlso": [ - "http://freeimage.sourceforge.net/freeimage-license.txt" - ], - "isOsiApproved": false - }, - { - "reference": "https://spdx.org/licenses/iMatix.html", + "reference": "https://spdx.org/licenses/PHP-3.01.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/iMatix.json", - "referenceNumber": 399, - "name": "iMatix Standard Function Library Agreement", - "licenseId": "iMatix", + "detailsUrl": "https://spdx.org/licenses/PHP-3.01.json", + "referenceNumber": 469, + "name": "PHP License v3.01", + "licenseId": "PHP-3.01", "seeAlso": [ - "http://legacy.imatix.com/html/sfl/sfl4.htm#license" + "http://www.php.net/license/3_01.txt" ], - "isOsiApproved": false, + "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/TOSL.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TOSL.json", - "referenceNumber": 400, - "name": "Trusster Open Source License", - "licenseId": "TOSL", - "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/TOSL" - ], - "isOsiApproved": false - }, - { - "reference": "https://spdx.org/licenses/MIT-CMU.html", + "reference": "https://spdx.org/licenses/Plexus.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-CMU.json", - "referenceNumber": 401, - "name": "CMU License", - "licenseId": "MIT-CMU", + "detailsUrl": "https://spdx.org/licenses/Plexus.json", + "referenceNumber": 472, + "name": "Plexus Classworlds License", + "licenseId": "Plexus", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing:MIT?rd\u003dLicensing/MIT#CMU_Style", - "https://github.com/python-pillow/Pillow/blob/fffb426092c8db24a5f4b6df243a8a3c01fb63cd/LICENSE" + "https://fedoraproject.org/wiki/Licensing/Plexus_Classworlds_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CATOSL-1.1.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CATOSL-1.1.json", - "referenceNumber": 402, - "name": "Computer Associates Trusted Open Source License 1.1", - "licenseId": "CATOSL-1.1", - "seeAlso": [ - "https://opensource.org/licenses/CATOSL-1.1" - ], - "isOsiApproved": true - }, - { - "reference": "https://spdx.org/licenses/LPPL-1.1.html", + "reference": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LPPL-1.1.json", - "referenceNumber": 403, - "name": "LaTeX Project Public License v1.1", - "licenseId": "LPPL-1.1", + "detailsUrl": "https://spdx.org/licenses/PolyForm-Noncommercial-1.0.0.json", + "referenceNumber": 188, + "name": "PolyForm Noncommercial License 1.0.0", + "licenseId": "PolyForm-Noncommercial-1.0.0", "seeAlso": [ - "http://www.latex-project.org/lppl/lppl-1-1.txt" + "https://polyformproject.org/licenses/noncommercial/1.0.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.html", + "reference": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json", - "referenceNumber": 404, - "name": "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", - "licenseId": "CC-BY-NC-SA-2.0-FR", + "detailsUrl": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.json", + "referenceNumber": 302, + "name": "PolyForm Small Business License 1.0.0", + "licenseId": "PolyForm-Small-Business-1.0.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.0/fr/legalcode" + "https://polyformproject.org/licenses/small-business/1.0.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.1-or-later.html", + "reference": "https://spdx.org/licenses/PostgreSQL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.1-or-later.json", - "referenceNumber": 405, - "name": "GNU Lesser General Public License v2.1 or later", - "licenseId": "LGPL-2.1-or-later", + "detailsUrl": "https://spdx.org/licenses/PostgreSQL.json", + "referenceNumber": 237, + "name": "PostgreSQL License", + "licenseId": "PostgreSQL", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", - "https://opensource.org/licenses/LGPL-2.1" + "http://www.postgresql.org/about/licence", + "https://opensource.org/licenses/PostgreSQL" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Arphic-1999.html", + "reference": "https://spdx.org/licenses/PSF-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Arphic-1999.json", - "referenceNumber": 406, - "name": "Arphic Public License", - "licenseId": "Arphic-1999", + "detailsUrl": "https://spdx.org/licenses/PSF-2.0.json", + "referenceNumber": 452, + "name": "Python Software Foundation License 2.0", + "licenseId": "PSF-2.0", "seeAlso": [ - "http://ftp.gnu.org/gnu/non-gnu/chinese-fonts-truetype/LICENSE" + "https://opensource.org/licenses/Python-2.0" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Sendmail-8.23.html", + "reference": "https://spdx.org/licenses/psfrag.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Sendmail-8.23.json", - "referenceNumber": 407, - "name": "Sendmail License 8.23", - "licenseId": "Sendmail-8.23", + "detailsUrl": "https://spdx.org/licenses/psfrag.json", + "referenceNumber": 268, + "name": "psfrag License", + "licenseId": "psfrag", "seeAlso": [ - "https://www.proofpoint.com/sites/default/files/sendmail-license.pdf", - "https://web.archive.org/web/20181003101040/https://www.proofpoint.com/sites/default/files/sendmail-license.pdf" + "https://fedoraproject.org/wiki/Licensing/psfrag" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CECILL-2.0.html", + "reference": "https://spdx.org/licenses/psutils.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CECILL-2.0.json", - "referenceNumber": 408, - "name": "CeCILL Free Software License Agreement v2.0", - "licenseId": "CECILL-2.0", + "detailsUrl": "https://spdx.org/licenses/psutils.json", + "referenceNumber": 106, + "name": "psutils License", + "licenseId": "psutils", "seeAlso": [ - "http://www.cecill.info/licences/Licence_CeCILL_V2-en.html" + "https://fedoraproject.org/wiki/Licensing/psutils" ], - "isOsiApproved": false, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.html", + "reference": "https://spdx.org/licenses/Python-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-3.0.json", - "referenceNumber": 409, - "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", - "licenseId": "CC-BY-NC-ND-3.0", + "detailsUrl": "https://spdx.org/licenses/Python-2.0.json", + "referenceNumber": 368, + "name": "Python License 2.0", + "licenseId": "Python-2.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/3.0/legalcode" + "https://opensource.org/licenses/Python-2.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/APSL-2.0.html", + "reference": "https://spdx.org/licenses/Python-2.0.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/APSL-2.0.json", - "referenceNumber": 410, - "name": "Apple Public Source License 2.0", - "licenseId": "APSL-2.0", + "detailsUrl": "https://spdx.org/licenses/Python-2.0.1.json", + "referenceNumber": 200, + "name": "Python License 2.0.1", + "licenseId": "Python-2.0.1", "seeAlso": [ - "http://www.opensource.apple.com/license/apsl/" + "https://www.python.org/download/releases/2.0.1/license/", + "https://docs.python.org/3/license.html", + "https://github.com/python/cpython/blob/main/LICENSE" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Unicode-TOU.html", + "reference": "https://spdx.org/licenses/Qhull.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Unicode-TOU.json", - "referenceNumber": 411, - "name": "Unicode Terms of Use", - "licenseId": "Unicode-TOU", + "detailsUrl": "https://spdx.org/licenses/Qhull.json", + "referenceNumber": 270, + "name": "Qhull License", + "licenseId": "Qhull", "seeAlso": [ - "http://www.unicode.org/copyright.html" + "https://fedoraproject.org/wiki/Licensing/Qhull" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/MS-RL.html", + "reference": "https://spdx.org/licenses/QPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MS-RL.json", - "referenceNumber": 412, - "name": "Microsoft Reciprocal License", - "licenseId": "MS-RL", + "detailsUrl": "https://spdx.org/licenses/QPL-1.0.json", + "referenceNumber": 57, + "name": "Q Public License 1.0", + "licenseId": "QPL-1.0", "seeAlso": [ - "http://www.microsoft.com/opensource/licenses.mspx", - "https://opensource.org/licenses/MS-RL" + "http://doc.qt.nokia.com/3.3/license.html", + "https://opensource.org/licenses/QPL-1.0", + "https://doc.qt.io/archives/3.3/license.html" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.html", + "reference": "https://spdx.org/licenses/Rdisc.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-ND-2.0.json", - "referenceNumber": 413, - "name": "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", - "licenseId": "CC-BY-NC-ND-2.0", + "detailsUrl": "https://spdx.org/licenses/Rdisc.json", + "referenceNumber": 14, + "name": "Rdisc License", + "licenseId": "Rdisc", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-nd/2.0/legalcode" + "https://fedoraproject.org/wiki/Licensing/Rdisc_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OSL-3.0.html", + "reference": "https://spdx.org/licenses/RHeCos-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSL-3.0.json", - "referenceNumber": 414, - "name": "Open Software License 3.0", - "licenseId": "OSL-3.0", + "detailsUrl": "https://spdx.org/licenses/RHeCos-1.1.json", + "referenceNumber": 224, + "name": "Red Hat eCos Public License v1.1", + "licenseId": "RHeCos-1.1", "seeAlso": [ - "https://web.archive.org/web/20120101081418/http://rosenlaw.com:80/OSL3.0.htm", - "https://opensource.org/licenses/OSL-3.0" + "http://ecos.sourceware.org/old-license.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/Nokia.html", + "reference": "https://spdx.org/licenses/RPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Nokia.json", - "referenceNumber": 415, - "name": "Nokia Open Source License", - "licenseId": "Nokia", + "detailsUrl": "https://spdx.org/licenses/RPL-1.1.json", + "referenceNumber": 54, + "name": "Reciprocal Public License 1.1", + "licenseId": "RPL-1.1", "seeAlso": [ - "https://opensource.org/licenses/nokia" + "https://opensource.org/licenses/RPL-1.1" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/MPL-1.1.html", + "reference": "https://spdx.org/licenses/RPL-1.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MPL-1.1.json", - "referenceNumber": 416, - "name": "Mozilla Public License 1.1", - "licenseId": "MPL-1.1", - "seeAlso": [ - "http://www.mozilla.org/MPL/MPL-1.1.html", - "https://opensource.org/licenses/MPL-1.1" - ], - "isOsiApproved": true, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/GPL-1.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-1.0.json", - "referenceNumber": 417, - "name": "GNU General Public License v1.0 only", - "licenseId": "GPL-1.0", + "detailsUrl": "https://spdx.org/licenses/RPL-1.5.json", + "referenceNumber": 36, + "name": "Reciprocal Public License 1.5", + "licenseId": "RPL-1.5", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html" + "https://opensource.org/licenses/RPL-1.5" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-1.0.html", + "reference": "https://spdx.org/licenses/RPSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-1.0.json", - "referenceNumber": 418, - "name": "Creative Commons Attribution Non Commercial 1.0 Generic", - "licenseId": "CC-BY-NC-1.0", + "detailsUrl": "https://spdx.org/licenses/RPSL-1.0.json", + "referenceNumber": 324, + "name": "RealNetworks Public Source License v1.0", + "licenseId": "RPSL-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc/1.0/legalcode" + "https://helixcommunity.org/content/rpsl", + "https://opensource.org/licenses/RPSL-1.0" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CERN-OHL-1.1.html", + "reference": "https://spdx.org/licenses/RSA-MD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.1.json", - "referenceNumber": 419, - "name": "CERN Open Hardware Licence v1.1", - "licenseId": "CERN-OHL-1.1", + "detailsUrl": "https://spdx.org/licenses/RSA-MD.json", + "referenceNumber": 47, + "name": "RSA Message-Digest License", + "licenseId": "RSA-MD", "seeAlso": [ - "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1" + "http://www.faqs.org/rfcs/rfc1321.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/W3C-20150513.html", + "reference": "https://spdx.org/licenses/RSCPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/W3C-20150513.json", - "referenceNumber": 420, - "name": "W3C Software Notice and Document License (2015-05-13)", - "licenseId": "W3C-20150513", + "detailsUrl": "https://spdx.org/licenses/RSCPL.json", + "referenceNumber": 218, + "name": "Ricoh Source Code Public License", + "licenseId": "RSCPL", "seeAlso": [ - "https://www.w3.org/Consortium/Legal/2015/copyright-software-and-document" + "http://wayback.archive.org/web/20060715140826/http://www.risource.org/RPL/RPL-1.0A.shtml", + "https://opensource.org/licenses/RSCPL" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/SPL-1.0.html", + "reference": "https://spdx.org/licenses/Ruby.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SPL-1.0.json", - "referenceNumber": 421, - "name": "Sun Public License v1.0", - "licenseId": "SPL-1.0", + "detailsUrl": "https://spdx.org/licenses/Ruby.json", + "referenceNumber": 395, + "name": "Ruby License", + "licenseId": "Ruby", "seeAlso": [ - "https://opensource.org/licenses/SPL-1.0" + "http://www.ruby-lang.org/en/LICENSE.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LZMA-SDK-9.22.html", + "reference": "https://spdx.org/licenses/SAX-PD.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LZMA-SDK-9.22.json", - "referenceNumber": 422, - "name": "LZMA SDK License (versions 9.22 and beyond)", - "licenseId": "LZMA-SDK-9.22", + "detailsUrl": "https://spdx.org/licenses/SAX-PD.json", + "referenceNumber": 67, + "name": "Sax Public Domain Notice", + "licenseId": "SAX-PD", "seeAlso": [ - "https://www.7-zip.org/sdk.html", - "https://sourceforge.net/projects/sevenzip/files/LZMA%20SDK/" + "http://www.saxproject.org/copying.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPLLR.html", + "reference": "https://spdx.org/licenses/Saxpath.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LGPLLR.json", - "referenceNumber": 423, - "name": "Lesser General Public License For Linguistic Resources", - "licenseId": "LGPLLR", + "detailsUrl": "https://spdx.org/licenses/Saxpath.json", + "referenceNumber": 127, + "name": "Saxpath License", + "licenseId": "Saxpath", "seeAlso": [ - "http://www-igm.univ-mlv.fr/~unitex/lgpllr.html" + "https://fedoraproject.org/wiki/Licensing/Saxpath_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LiLiQ-R-1.1.html", + "reference": "https://spdx.org/licenses/SCEA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LiLiQ-R-1.1.json", - "referenceNumber": 424, - "name": "Licence Libre du Québec – Réciprocité version 1.1", - "licenseId": "LiLiQ-R-1.1", + "detailsUrl": "https://spdx.org/licenses/SCEA.json", + "referenceNumber": 95, + "name": "SCEA Shared Source License", + "licenseId": "SCEA", "seeAlso": [ - "https://www.forge.gouv.qc.ca/participez/licence-logicielle/licence-libre-du-quebec-liliq-en-francais/licence-libre-du-quebec-reciprocite-liliq-r-v1-1/", - "http://opensource.org/licenses/LiLiQ-R-1.1" + "http://research.scea.com/scea_shared_source_license.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ZPL-2.0.html", + "reference": "https://spdx.org/licenses/SchemeReport.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ZPL-2.0.json", - "referenceNumber": 425, - "name": "Zope Public License 2.0", - "licenseId": "ZPL-2.0", - "seeAlso": [ - "http://old.zope.org/Resources/License/ZPL-2.0", - "https://opensource.org/licenses/ZPL-2.0" - ], - "isOsiApproved": true, - "isFsfLibre": true + "detailsUrl": "https://spdx.org/licenses/SchemeReport.json", + "referenceNumber": 178, + "name": "Scheme Language Report License", + "licenseId": "SchemeReport", + "seeAlso": [], + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.html", + "reference": "https://spdx.org/licenses/Sendmail.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-No-Nuclear-Warranty.json", - "referenceNumber": 426, - "name": "BSD 3-Clause No Nuclear Warranty", - "licenseId": "BSD-3-Clause-No-Nuclear-Warranty", + "detailsUrl": "https://spdx.org/licenses/Sendmail.json", + "referenceNumber": 422, + "name": "Sendmail License", + "licenseId": "Sendmail", "seeAlso": [ - "https://jogamp.org/git/?p\u003dgluegen.git;a\u003dblob_plain;f\u003dLICENSE.txt" + "http://www.sendmail.com/pdfs/open_source/sendmail_license.pdf", + "https://web.archive.org/web/20160322142305/https://www.sendmail.com/pdfs/open_source/sendmail_license.pdf" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Spencer-94.html", + "reference": "https://spdx.org/licenses/Sendmail-8.23.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Spencer-94.json", - "referenceNumber": 427, - "name": "Spencer License 94", - "licenseId": "Spencer-94", + "detailsUrl": "https://spdx.org/licenses/Sendmail-8.23.json", + "referenceNumber": 500, + "name": "Sendmail License 8.23", + "licenseId": "Sendmail-8.23", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" + "https://www.proofpoint.com/sites/default/files/sendmail-license.pdf", + "https://web.archive.org/web/20181003101040/https://www.proofpoint.com/sites/default/files/sendmail-license.pdf" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-SA-2.5.html", + "reference": "https://spdx.org/licenses/SGI-B-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-SA-2.5.json", - "referenceNumber": 428, - "name": "Creative Commons Attribution Share Alike 2.5 Generic", - "licenseId": "CC-BY-SA-2.5", + "detailsUrl": "https://spdx.org/licenses/SGI-B-1.0.json", + "referenceNumber": 93, + "name": "SGI Free Software License B v1.0", + "licenseId": "SGI-B-1.0", "seeAlso": [ - "https://creativecommons.org/licenses/by-sa/2.5/legalcode" + "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OSET-PL-2.1.html", + "reference": "https://spdx.org/licenses/SGI-B-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OSET-PL-2.1.json", - "referenceNumber": 429, - "name": "OSET Public License version 2.1", - "licenseId": "OSET-PL-2.1", + "detailsUrl": "https://spdx.org/licenses/SGI-B-1.1.json", + "referenceNumber": 68, + "name": "SGI Free Software License B v1.1", + "licenseId": "SGI-B-1.1", "seeAlso": [ - "http://www.osetfoundation.org/public-license", - "https://opensource.org/licenses/OPL-2.1" + "http://oss.sgi.com/projects/FreeB/" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/IPA.html", + "reference": "https://spdx.org/licenses/SGI-B-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IPA.json", - "referenceNumber": 430, - "name": "IPA Font License", - "licenseId": "IPA", + "detailsUrl": "https://spdx.org/licenses/SGI-B-2.0.json", + "referenceNumber": 352, + "name": "SGI Free Software License B v2.0", + "licenseId": "SGI-B-2.0", "seeAlso": [ - "https://opensource.org/licenses/IPA" + "http://oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Glide.html", + "reference": "https://spdx.org/licenses/SHL-0.5.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Glide.json", - "referenceNumber": 431, - "name": "3dfx Glide License", - "licenseId": "Glide", + "detailsUrl": "https://spdx.org/licenses/SHL-0.5.json", + "referenceNumber": 88, + "name": "Solderpad Hardware License v0.5", + "licenseId": "SHL-0.5", "seeAlso": [ - "http://www.users.on.net/~triforce/glidexp/COPYING.txt" + "https://solderpad.org/licenses/SHL-0.5/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/RSA-MD.html", + "reference": "https://spdx.org/licenses/SHL-0.51.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RSA-MD.json", - "referenceNumber": 432, - "name": "RSA Message-Digest License", - "licenseId": "RSA-MD", + "detailsUrl": "https://spdx.org/licenses/SHL-0.51.json", + "referenceNumber": 345, + "name": "Solderpad Hardware License, Version 0.51", + "licenseId": "SHL-0.51", "seeAlso": [ - "http://www.faqs.org/rfcs/rfc1321.html" + "https://solderpad.org/licenses/SHL-0.51/" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CPAL-1.0.html", + "reference": "https://spdx.org/licenses/SimPL-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CPAL-1.0.json", - "referenceNumber": 433, - "name": "Common Public Attribution License 1.0", - "licenseId": "CPAL-1.0", + "detailsUrl": "https://spdx.org/licenses/SimPL-2.0.json", + "referenceNumber": 45, + "name": "Simple Public License 2.0", + "licenseId": "SimPL-2.0", "seeAlso": [ - "https://opensource.org/licenses/CPAL-1.0" + "https://opensource.org/licenses/SimPL-2.0" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/UPL-1.0.html", + "reference": "https://spdx.org/licenses/SISSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/UPL-1.0.json", - "referenceNumber": 434, - "name": "Universal Permissive License v1.0", - "licenseId": "UPL-1.0", + "detailsUrl": "https://spdx.org/licenses/SISSL.json", + "referenceNumber": 86, + "name": "Sun Industry Standards Source License v1.1", + "licenseId": "SISSL", "seeAlso": [ - "https://opensource.org/licenses/UPL" + "http://www.openoffice.org/licenses/sissl_license.html", + "https://opensource.org/licenses/SISSL" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-2.0.html", + "reference": "https://spdx.org/licenses/SISSL-1.2.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-2.0.json", - "referenceNumber": 435, - "name": "Creative Commons Attribution No Derivatives 2.0 Generic", - "licenseId": "CC-BY-ND-2.0", + "detailsUrl": "https://spdx.org/licenses/SISSL-1.2.json", + "referenceNumber": 154, + "name": "Sun Industry Standards Source License v1.2", + "licenseId": "SISSL-1.2", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/2.0/legalcode" + "http://gridscheduler.sourceforge.net/Gridengine_SISSL_license.html" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/libtiff.html", + "reference": "https://spdx.org/licenses/Sleepycat.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/libtiff.json", - "referenceNumber": 436, - "name": "libtiff License", - "licenseId": "libtiff", + "detailsUrl": "https://spdx.org/licenses/Sleepycat.json", + "referenceNumber": 261, + "name": "Sleepycat License", + "licenseId": "Sleepycat", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/libtiff" + "https://opensource.org/licenses/Sleepycat" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/zlib-acknowledgement.html", + "reference": "https://spdx.org/licenses/SMLNJ.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/zlib-acknowledgement.json", - "referenceNumber": 437, - "name": "zlib/libpng License with Acknowledgement", - "licenseId": "zlib-acknowledgement", + "detailsUrl": "https://spdx.org/licenses/SMLNJ.json", + "referenceNumber": 184, + "name": "Standard ML of New Jersey License", + "licenseId": "SMLNJ", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/ZlibWithAcknowledgement" + "https://www.smlnj.org/license.html" + ], + "isOsiApproved": false, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/SMPPL.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/SMPPL.json", + "referenceNumber": 228, + "name": "Secure Messaging Protocol Public License", + "licenseId": "SMPPL", + "seeAlso": [ + "https://github.com/dcblake/SMP/blob/master/Documentation/License.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-Source-Code.html", + "reference": "https://spdx.org/licenses/SNIA.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-Source-Code.json", - "referenceNumber": 438, - "name": "BSD Source Code Attribution", - "licenseId": "BSD-Source-Code", + "detailsUrl": "https://spdx.org/licenses/SNIA.json", + "referenceNumber": 104, + "name": "SNIA Public License 1.1", + "licenseId": "SNIA", "seeAlso": [ - "https://github.com/robbiehanson/CocoaHTTPServer/blob/master/LICENSE.txt" + "https://fedoraproject.org/wiki/Licensing/SNIA_Public_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BitTorrent-1.0.html", + "reference": "https://spdx.org/licenses/Spencer-86.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BitTorrent-1.0.json", - "referenceNumber": 439, - "name": "BitTorrent Open Source License v1.0", - "licenseId": "BitTorrent-1.0", + "detailsUrl": "https://spdx.org/licenses/Spencer-86.json", + "referenceNumber": 351, + "name": "Spencer License 86", + "licenseId": "Spencer-86", "seeAlso": [ - "http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1\u003d1.1\u0026r2\u003d1.1.1.1\u0026diff_format\u003ds" + "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/HPND-sell-variant.html", + "reference": "https://spdx.org/licenses/Spencer-94.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/HPND-sell-variant.json", - "referenceNumber": 440, - "name": "Historical Permission Notice and Disclaimer - sell variant", - "licenseId": "HPND-sell-variant", + "detailsUrl": "https://spdx.org/licenses/Spencer-94.json", + "referenceNumber": 144, + "name": "Spencer License 94", + "licenseId": "Spencer-94", "seeAlso": [ - "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h\u003dv4.19" + "https://fedoraproject.org/wiki/Licensing/Henry_Spencer_Reg-Ex_Library_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/W3C-19980720.html", + "reference": "https://spdx.org/licenses/Spencer-99.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/W3C-19980720.json", - "referenceNumber": 441, - "name": "W3C Software Notice and License (1998-07-20)", - "licenseId": "W3C-19980720", + "detailsUrl": "https://spdx.org/licenses/Spencer-99.json", + "referenceNumber": 233, + "name": "Spencer License 99", + "licenseId": "Spencer-99", "seeAlso": [ - "http://www.w3.org/Consortium/Legal/copyright-software-19980720.html" + "http://www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CERN-OHL-1.2.html", + "reference": "https://spdx.org/licenses/SPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-1.2.json", - "referenceNumber": 442, - "name": "CERN Open Hardware Licence v1.2", - "licenseId": "CERN-OHL-1.2", + "detailsUrl": "https://spdx.org/licenses/SPL-1.0.json", + "referenceNumber": 124, + "name": "Sun Public License v1.0", + "licenseId": "SPL-1.0", "seeAlso": [ - "https://www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2" + "https://opensource.org/licenses/SPL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OFL-1.0-RFN.html", + "reference": "https://spdx.org/licenses/SSH-OpenSSH.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OFL-1.0-RFN.json", - "referenceNumber": 443, - "name": "SIL Open Font License 1.0 with Reserved Font Name", - "licenseId": "OFL-1.0-RFN", + "detailsUrl": "https://spdx.org/licenses/SSH-OpenSSH.json", + "referenceNumber": 494, + "name": "SSH OpenSSH license", + "licenseId": "SSH-OpenSSH", "seeAlso": [ - "http://scripts.sil.org/cms/scripts/page.php?item_id\u003dOFL10_web" + "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-PDDC.html", + "reference": "https://spdx.org/licenses/SSH-short.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-PDDC.json", - "referenceNumber": 444, - "name": "Creative Commons Public Domain Dedication and Certification", - "licenseId": "CC-PDDC", + "detailsUrl": "https://spdx.org/licenses/SSH-short.json", + "referenceNumber": 165, + "name": "SSH short notice", + "licenseId": "SSH-short", "seeAlso": [ - "https://creativecommons.org/licenses/publicdomain/" + "https://github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h", + "http://web.mit.edu/kolya/.f/root/athena.mit.edu/sipb.mit.edu/project/openssh/OldFiles/src/openssh-2.9.9p2/ssh-add.1", + "https://joinup.ec.europa.eu/svn/lesoll/trunk/italc/lib/src/dsa_key.cpp" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.html", + "reference": "https://spdx.org/licenses/SSPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/PolyForm-Small-Business-1.0.0.json", - "referenceNumber": 445, - "name": "PolyForm Small Business License 1.0.0", - "licenseId": "PolyForm-Small-Business-1.0.0", + "detailsUrl": "https://spdx.org/licenses/SSPL-1.0.json", + "referenceNumber": 77, + "name": "Server Side Public License, v 1", + "licenseId": "SSPL-1.0", "seeAlso": [ - "https://polyformproject.org/licenses/small-business/1.0.0" + "https://www.mongodb.com/licensing/server-side-public-license" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Watcom-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Watcom-1.0.json", - "referenceNumber": 446, - "name": "Sybase Open Watcom Public License 1.0", - "licenseId": "Watcom-1.0", + "reference": "https://spdx.org/licenses/StandardML-NJ.html", + "isDeprecatedLicenseId": true, + "detailsUrl": "https://spdx.org/licenses/StandardML-NJ.json", + "referenceNumber": 501, + "name": "Standard ML of New Jersey License", + "licenseId": "StandardML-NJ", "seeAlso": [ - "https://opensource.org/licenses/Watcom-1.0" + "https://www.smlnj.org/license.html" ], - "isOsiApproved": true, - "isFsfLibre": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/CC-BY-2.0.html", + "reference": "https://spdx.org/licenses/SugarCRM-1.1.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-2.0.json", - "referenceNumber": 447, - "name": "Creative Commons Attribution 2.0 Generic", - "licenseId": "CC-BY-2.0", + "detailsUrl": "https://spdx.org/licenses/SugarCRM-1.1.3.json", + "referenceNumber": 91, + "name": "SugarCRM Public License v1.1.3", + "licenseId": "SugarCRM-1.1.3", "seeAlso": [ - "https://creativecommons.org/licenses/by/2.0/legalcode" + "http://www.sugarcrm.com/crm/SPL" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/RPL-1.5.html", + "reference": "https://spdx.org/licenses/SWL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RPL-1.5.json", - "referenceNumber": 448, - "name": "Reciprocal Public License 1.5", - "licenseId": "RPL-1.5", + "detailsUrl": "https://spdx.org/licenses/SWL.json", + "referenceNumber": 293, + "name": "Scheme Widget Library (SWL) Software License Agreement", + "licenseId": "SWL", "seeAlso": [ - "https://opensource.org/licenses/RPL-1.5" + "https://fedoraproject.org/wiki/Licensing/SWL" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/NLOD-2.0.html", + "reference": "https://spdx.org/licenses/Symlinks.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NLOD-2.0.json", - "referenceNumber": 449, - "name": "Norwegian Licence for Open Government Data (NLOD) 2.0", - "licenseId": "NLOD-2.0", + "detailsUrl": "https://spdx.org/licenses/Symlinks.json", + "referenceNumber": 457, + "name": "Symlinks License", + "licenseId": "Symlinks", "seeAlso": [ - "http://data.norge.no/nlod/en/2.0" + "https://www.mail-archive.com/debian-bugs-rc@lists.debian.org/msg11494.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/copyleft-next-0.3.0.html", + "reference": "https://spdx.org/licenses/TAPR-OHL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/copyleft-next-0.3.0.json", - "referenceNumber": 450, - "name": "copyleft-next 0.3.0", - "licenseId": "copyleft-next-0.3.0", + "detailsUrl": "https://spdx.org/licenses/TAPR-OHL-1.0.json", + "referenceNumber": 230, + "name": "TAPR Open Hardware License v1.0", + "licenseId": "TAPR-OHL-1.0", "seeAlso": [ - "https://github.com/copyleft-next/copyleft-next/blob/master/Releases/copyleft-next-0.3.0" + "https://www.tapr.org/OHL" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/RPSL-1.0.html", + "reference": "https://spdx.org/licenses/TCL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/RPSL-1.0.json", - "referenceNumber": 451, - "name": "RealNetworks Public Source License v1.0", - "licenseId": "RPSL-1.0", + "detailsUrl": "https://spdx.org/licenses/TCL.json", + "referenceNumber": 94, + "name": "TCL/TK License", + "licenseId": "TCL", "seeAlso": [ - "https://helixcommunity.org/content/rpsl", - "https://opensource.org/licenses/RPSL-1.0" + "http://www.tcl.tk/software/tcltk/license.html", + "https://fedoraproject.org/wiki/Licensing/TCL" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-ND-1.0.html", + "reference": "https://spdx.org/licenses/TCP-wrappers.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-ND-1.0.json", - "referenceNumber": 452, - "name": "Creative Commons Attribution No Derivatives 1.0 Generic", - "licenseId": "CC-BY-ND-1.0", + "detailsUrl": "https://spdx.org/licenses/TCP-wrappers.json", + "referenceNumber": 416, + "name": "TCP Wrappers License", + "licenseId": "TCP-wrappers", "seeAlso": [ - "https://creativecommons.org/licenses/by-nd/1.0/legalcode" + "http://rc.quest.com/topics/openssh/license.php#tcpwrappers" ], - "isOsiApproved": false, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LiLiQ-P-1.1.html", + "reference": "https://spdx.org/licenses/TMate.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/LiLiQ-P-1.1.json", - "referenceNumber": 453, - "name": "Licence Libre du Québec – Permissive version 1.1", - "licenseId": "LiLiQ-P-1.1", + "detailsUrl": "https://spdx.org/licenses/TMate.json", + "referenceNumber": 295, + "name": "TMate Open Source License", + "licenseId": "TMate", "seeAlso": [ - "https://forge.gouv.qc.ca/licence/fr/liliq-v1-1/", - "http://opensource.org/licenses/LiLiQ-P-1.1" + "http://svnkit.com/license.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/FreeBSD-DOC.html", + "reference": "https://spdx.org/licenses/TORQUE-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/FreeBSD-DOC.json", - "referenceNumber": 454, - "name": "FreeBSD Documentation License", - "licenseId": "FreeBSD-DOC", + "detailsUrl": "https://spdx.org/licenses/TORQUE-1.1.json", + "referenceNumber": 421, + "name": "TORQUE v2.5+ Software License v1.1", + "licenseId": "TORQUE-1.1", "seeAlso": [ - "https://www.freebsd.org/copyright/freebsd-doc-license/" + "https://fedoraproject.org/wiki/Licensing/TORQUEv1.1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Afmparse.html", + "reference": "https://spdx.org/licenses/TOSL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Afmparse.json", - "referenceNumber": 455, - "name": "Afmparse License", - "licenseId": "Afmparse", + "detailsUrl": "https://spdx.org/licenses/TOSL.json", + "referenceNumber": 304, + "name": "Trusster Open Source License", + "licenseId": "TOSL", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Afmparse" + "https://fedoraproject.org/wiki/Licensing/TOSL" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/VSL-1.0.html", + "reference": "https://spdx.org/licenses/TPDL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/VSL-1.0.json", - "referenceNumber": 456, - "name": "Vovida Software License v1.0", - "licenseId": "VSL-1.0", + "detailsUrl": "https://spdx.org/licenses/TPDL.json", + "referenceNumber": 41, + "name": "Time::ParseDate License", + "licenseId": "TPDL", "seeAlso": [ - "https://opensource.org/licenses/VSL-1.0" + "https://metacpan.org/pod/Time::ParseDate#LICENSE" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.html", + "reference": "https://spdx.org/licenses/TTWL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-NC-SA-2.5.json", - "referenceNumber": 457, - "name": "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", - "licenseId": "CC-BY-NC-SA-2.5", + "detailsUrl": "https://spdx.org/licenses/TTWL.json", + "referenceNumber": 251, + "name": "Text-Tabs+Wrap License", + "licenseId": "TTWL", "seeAlso": [ - "https://creativecommons.org/licenses/by-nc-sa/2.5/legalcode" + "https://fedoraproject.org/wiki/Licensing/TTWL", + "https://github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Jam.html", + "reference": "https://spdx.org/licenses/TU-Berlin-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Jam.json", - "referenceNumber": 458, - "name": "Jam License", - "licenseId": "Jam", + "detailsUrl": "https://spdx.org/licenses/TU-Berlin-1.0.json", + "referenceNumber": 417, + "name": "Technische Universitaet Berlin License 1.0", + "licenseId": "TU-Berlin-1.0", "seeAlso": [ - "https://www.boost.org/doc/libs/1_35_0/doc/html/jam.html", - "https://web.archive.org/web/20160330173339/https://swarm.workshop.perforce.com/files/guest/perforce_software/jam/src/README" + "https://github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OML.html", + "reference": "https://spdx.org/licenses/TU-Berlin-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OML.json", - "referenceNumber": 459, - "name": "Open Market License", - "licenseId": "OML", + "detailsUrl": "https://spdx.org/licenses/TU-Berlin-2.0.json", + "referenceNumber": 61, + "name": "Technische Universitaet Berlin License 2.0", + "licenseId": "TU-Berlin-2.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Open_Market_License" + "https://github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/ADSL.html", + "reference": "https://spdx.org/licenses/UCL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/ADSL.json", - "referenceNumber": 460, - "name": "Amazon Digital Services License", - "licenseId": "ADSL", + "detailsUrl": "https://spdx.org/licenses/UCL-1.0.json", + "referenceNumber": 201, + "name": "Upstream Compatibility License v1.0", + "licenseId": "UCL-1.0", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/AmazonDigitalServicesLicense" + "https://opensource.org/licenses/UCL-1.0" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Zed.html", + "reference": "https://spdx.org/licenses/Unicode-DFS-2015.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Zed.json", - "referenceNumber": 461, - "name": "Zed License", - "licenseId": "Zed", + "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2015.json", + "referenceNumber": 332, + "name": "Unicode License Agreement - Data Files and Software (2015)", + "licenseId": "Unicode-DFS-2015", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Zed" + "https://web.archive.org/web/20151224134844/http://unicode.org/copyright.html" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Bitstream-Vera.html", + "reference": "https://spdx.org/licenses/Unicode-DFS-2016.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Bitstream-Vera.json", - "referenceNumber": 462, - "name": "Bitstream Vera Font License", - "licenseId": "Bitstream-Vera", + "detailsUrl": "https://spdx.org/licenses/Unicode-DFS-2016.json", + "referenceNumber": 134, + "name": "Unicode License Agreement - Data Files and Software (2016)", + "licenseId": "Unicode-DFS-2016", "seeAlso": [ - "https://web.archive.org/web/20080207013128/http://www.gnome.org/fonts/", - "https://docubrain.com/sites/default/files/licenses/bitstream-vera.html" + "http://www.unicode.org/copyright.html" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/IPL-1.0.html", + "reference": "https://spdx.org/licenses/Unicode-TOU.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/IPL-1.0.json", - "referenceNumber": 463, - "name": "IBM Public License v1.0", - "licenseId": "IPL-1.0", + "detailsUrl": "https://spdx.org/licenses/Unicode-TOU.json", + "referenceNumber": 459, + "name": "Unicode Terms of Use", + "licenseId": "Unicode-TOU", "seeAlso": [ - "https://opensource.org/licenses/IPL-1.0" + "http://www.unicode.org/copyright.html" ], - "isOsiApproved": true, - "isFsfLibre": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSL-1.0.html", + "reference": "https://spdx.org/licenses/Unlicense.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSL-1.0.json", - "referenceNumber": 464, - "name": "Boost Software License 1.0", - "licenseId": "BSL-1.0", + "detailsUrl": "https://spdx.org/licenses/Unlicense.json", + "referenceNumber": 194, + "name": "The Unlicense", + "licenseId": "Unlicense", "seeAlso": [ - "http://www.boost.org/LICENSE_1_0.txt", - "https://opensource.org/licenses/BSL-1.0" + "https://unlicense.org/" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/MirOS.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MirOS.json", - "referenceNumber": 465, - "name": "The MirOS Licence", - "licenseId": "MirOS", - "seeAlso": [ - "https://opensource.org/licenses/MirOS" - ], - "isOsiApproved": true - }, - { - "reference": "https://spdx.org/licenses/W3C.html", + "reference": "https://spdx.org/licenses/UPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/W3C.json", - "referenceNumber": 466, - "name": "W3C Software Notice and License (2002-12-31)", - "licenseId": "W3C", + "detailsUrl": "https://spdx.org/licenses/UPL-1.0.json", + "referenceNumber": 443, + "name": "Universal Permissive License v1.0", + "licenseId": "UPL-1.0", "seeAlso": [ - "http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231.html", - "https://opensource.org/licenses/W3C" + "https://opensource.org/licenses/UPL" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.2-invariants-only.html", + "reference": "https://spdx.org/licenses/Vim.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.2-invariants-only.json", - "referenceNumber": 467, - "name": "GNU Free Documentation License v1.2 only - invariants", - "licenseId": "GFDL-1.2-invariants-only", + "detailsUrl": "https://spdx.org/licenses/Vim.json", + "referenceNumber": 128, + "name": "Vim License", + "licenseId": "Vim", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.2.txt" + "http://vimdoc.sourceforge.net/htmldoc/uganda.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { "reference": "https://spdx.org/licenses/VOSTROM.html", "isDeprecatedLicenseId": false, "detailsUrl": "https://spdx.org/licenses/VOSTROM.json", - "referenceNumber": 468, + "referenceNumber": 17, "name": "VOSTROM Public License for Open Source", "licenseId": "VOSTROM", "seeAlso": [ @@ -5898,361 +6079,345 @@ "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/CERN-OHL-S-2.0.html", + "reference": "https://spdx.org/licenses/VSL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CERN-OHL-S-2.0.json", - "referenceNumber": 469, - "name": "CERN Open Hardware Licence Version 2 - Strongly Reciprocal", - "licenseId": "CERN-OHL-S-2.0", + "detailsUrl": "https://spdx.org/licenses/VSL-1.0.json", + "referenceNumber": 162, + "name": "Vovida Software License v1.0", + "licenseId": "VSL-1.0", "seeAlso": [ - "https://www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2" + "https://opensource.org/licenses/VSL-1.0" ], "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Apache-1.1.html", + "reference": "https://spdx.org/licenses/W3C.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Apache-1.1.json", - "referenceNumber": 470, - "name": "Apache License 1.1", - "licenseId": "Apache-1.1", + "detailsUrl": "https://spdx.org/licenses/W3C.json", + "referenceNumber": 149, + "name": "W3C Software Notice and License (2002-12-31)", + "licenseId": "W3C", "seeAlso": [ - "http://apache.org/licenses/LICENSE-1.1", - "https://opensource.org/licenses/Apache-1.1" + "http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231.html", + "https://opensource.org/licenses/W3C" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NASA-1.3.html", + "reference": "https://spdx.org/licenses/W3C-19980720.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NASA-1.3.json", - "referenceNumber": 471, - "name": "NASA Open Source Agreement 1.3", - "licenseId": "NASA-1.3", + "detailsUrl": "https://spdx.org/licenses/W3C-19980720.json", + "referenceNumber": 315, + "name": "W3C Software Notice and License (1998-07-20)", + "licenseId": "W3C-19980720", "seeAlso": [ - "http://ti.arc.nasa.gov/opensource/nosa/", - "https://opensource.org/licenses/NASA-1.3" + "http://www.w3.org/Consortium/Legal/copyright-software-19980720.html" ], - "isOsiApproved": true, - "isFsfLibre": false + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SHL-0.51.html", + "reference": "https://spdx.org/licenses/W3C-20150513.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SHL-0.51.json", - "referenceNumber": 472, - "name": "Solderpad Hardware License, Version 0.51", - "licenseId": "SHL-0.51", + "detailsUrl": "https://spdx.org/licenses/W3C-20150513.json", + "referenceNumber": 226, + "name": "W3C Software Notice and Document License (2015-05-13)", + "licenseId": "W3C-20150513", "seeAlso": [ - "https://solderpad.org/licenses/SHL-0.51/" + "https://www.w3.org/Consortium/Legal/2015/copyright-software-and-document" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/OPUBL-1.0.html", + "reference": "https://spdx.org/licenses/Watcom-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OPUBL-1.0.json", - "referenceNumber": 473, - "name": "Open Publication License v1.0", - "licenseId": "OPUBL-1.0", + "detailsUrl": "https://spdx.org/licenses/Watcom-1.0.json", + "referenceNumber": 227, + "name": "Sybase Open Watcom Public License 1.0", + "licenseId": "Watcom-1.0", "seeAlso": [ - "http://opencontent.org/openpub/", - "https://www.debian.org/opl", - "https://www.ctan.org/license/opl" + "https://opensource.org/licenses/Watcom-1.0" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": false }, { - "reference": "https://spdx.org/licenses/OCLC-2.0.html", + "reference": "https://spdx.org/licenses/Wsuipa.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OCLC-2.0.json", - "referenceNumber": 474, - "name": "OCLC Research Public License 2.0", - "licenseId": "OCLC-2.0", + "detailsUrl": "https://spdx.org/licenses/Wsuipa.json", + "referenceNumber": 157, + "name": "Wsuipa License", + "licenseId": "Wsuipa", "seeAlso": [ - "http://www.oclc.org/research/activities/software/license/v2final.htm", - "https://opensource.org/licenses/OCLC-2.0" + "https://fedoraproject.org/wiki/Licensing/Wsuipa" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.html", + "reference": "https://spdx.org/licenses/WTFPL.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-3-Clause-Open-MPI.json", - "referenceNumber": 475, - "name": "BSD 3-Clause Open MPI variant", - "licenseId": "BSD-3-Clause-Open-MPI", + "detailsUrl": "https://spdx.org/licenses/WTFPL.json", + "referenceNumber": 16, + "name": "Do What The F*ck You Want To Public License", + "licenseId": "WTFPL", "seeAlso": [ - "https://www.open-mpi.org/community/license.php", - "http://www.netlib.org/lapack/LICENSE.txt" + "http://www.wtfpl.net/about/", + "http://sam.zoy.org/wtfpl/COPYING" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.html", + "reference": "https://spdx.org/licenses/wxWindows.html", "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/GPL-2.0-with-GCC-exception.json", - "referenceNumber": 476, - "name": "GNU General Public License v2.0 w/GCC Runtime Library exception", - "licenseId": "GPL-2.0-with-GCC-exception", + "detailsUrl": "https://spdx.org/licenses/wxWindows.json", + "referenceNumber": 383, + "name": "wxWindows Library License", + "licenseId": "wxWindows", "seeAlso": [ - "https://gcc.gnu.org/git/?p\u003dgcc.git;a\u003dblob;f\u003dgcc/libgcc1.c;h\u003d762f5143fc6eed57b6797c82710f3538aa52b40b;hb\u003dcb143a3ce4fb417c68f5fa2691a1b1b1053dfba9#l10" + "https://opensource.org/licenses/WXwindows" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/Fair.html", + "reference": "https://spdx.org/licenses/X11.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Fair.json", - "referenceNumber": 477, - "name": "Fair License", - "licenseId": "Fair", + "detailsUrl": "https://spdx.org/licenses/X11.json", + "referenceNumber": 167, + "name": "X11 License", + "licenseId": "X11", "seeAlso": [ - "http://fairlicense.org/", - "https://opensource.org/licenses/Fair" + "http://www.xfree86.org/3.3.6/COPYRIGHT2.html#3" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.html", + "reference": "https://spdx.org/licenses/X11-distribute-modifications-variant.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.1-no-invariants-or-later.json", - "referenceNumber": 478, - "name": "GNU Free Documentation License v1.1 or later - no invariants", - "licenseId": "GFDL-1.1-no-invariants-or-later", + "detailsUrl": "https://spdx.org/licenses/X11-distribute-modifications-variant.json", + "referenceNumber": 217, + "name": "X11 License Distribution Modification Variant", + "licenseId": "X11-distribute-modifications-variant", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/fdl-1.1.txt" + "https://github.com/mirror/ncurses/blob/master/COPYING" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Saxpath.html", + "reference": "https://spdx.org/licenses/Xerox.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Saxpath.json", - "referenceNumber": 479, - "name": "Saxpath License", - "licenseId": "Saxpath", + "detailsUrl": "https://spdx.org/licenses/Xerox.json", + "referenceNumber": 493, + "name": "Xerox License", + "licenseId": "Xerox", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Saxpath_License" + "https://fedoraproject.org/wiki/Licensing/Xerox" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/SimPL-2.0.html", + "reference": "https://spdx.org/licenses/XFree86-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/SimPL-2.0.json", - "referenceNumber": 480, - "name": "Simple Public License 2.0", - "licenseId": "SimPL-2.0", + "detailsUrl": "https://spdx.org/licenses/XFree86-1.1.json", + "referenceNumber": 378, + "name": "XFree86 License 1.1", + "licenseId": "XFree86-1.1", "seeAlso": [ - "https://opensource.org/licenses/SimPL-2.0" + "http://www.xfree86.org/current/LICENSE4.html" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LGPL-2.0+.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.0+.json", - "referenceNumber": 481, - "name": "GNU Library General Public License v2 or later", - "licenseId": "LGPL-2.0+", + "reference": "https://spdx.org/licenses/xinetd.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/xinetd.json", + "referenceNumber": 343, + "name": "xinetd License", + "licenseId": "xinetd", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html" + "https://fedoraproject.org/wiki/Licensing/Xinetd_License" ], - "isOsiApproved": true + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/OLDAP-2.5.html", + "reference": "https://spdx.org/licenses/Xnet.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/OLDAP-2.5.json", - "referenceNumber": 482, - "name": "Open LDAP Public License v2.5", - "licenseId": "OLDAP-2.5", + "detailsUrl": "https://spdx.org/licenses/Xnet.json", + "referenceNumber": 82, + "name": "X.Net License", + "licenseId": "Xnet", "seeAlso": [ - "http://www.openldap.org/devel/gitweb.cgi?p\u003dopenldap.git;a\u003dblob;f\u003dLICENSE;hb\u003d6852b9d90022e8593c98205413380536b1b5a7cf" + "https://opensource.org/licenses/Xnet" ], - "isOsiApproved": false + "isOsiApproved": true }, { - "reference": "https://spdx.org/licenses/MIT-feh.html", + "reference": "https://spdx.org/licenses/xpp.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT-feh.json", - "referenceNumber": 483, - "name": "feh License", - "licenseId": "MIT-feh", + "detailsUrl": "https://spdx.org/licenses/xpp.json", + "referenceNumber": 425, + "name": "XPP License", + "licenseId": "xpp", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT#feh" + "https://fedoraproject.org/wiki/Licensing/xpp" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Barr.html", + "reference": "https://spdx.org/licenses/XSkat.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Barr.json", - "referenceNumber": 484, - "name": "Barr License", - "licenseId": "Barr", + "detailsUrl": "https://spdx.org/licenses/XSkat.json", + "referenceNumber": 146, + "name": "XSkat License", + "licenseId": "XSkat", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/Barr" + "https://fedoraproject.org/wiki/Licensing/XSkat_License" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Multics.html", + "reference": "https://spdx.org/licenses/YPL-1.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Multics.json", - "referenceNumber": 485, - "name": "Multics License", - "licenseId": "Multics", + "detailsUrl": "https://spdx.org/licenses/YPL-1.0.json", + "referenceNumber": 318, + "name": "Yahoo! Public License v1.0", + "licenseId": "YPL-1.0", "seeAlso": [ - "https://opensource.org/licenses/Multics" + "http://www.zimbra.com/license/yahoo_public_license_1.0.html" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Adobe-Glyph.html", + "reference": "https://spdx.org/licenses/YPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Adobe-Glyph.json", - "referenceNumber": 486, - "name": "Adobe Glyph List License", - "licenseId": "Adobe-Glyph", + "detailsUrl": "https://spdx.org/licenses/YPL-1.1.json", + "referenceNumber": 376, + "name": "Yahoo! Public License v1.1", + "licenseId": "YPL-1.1", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/MIT#AdobeGlyph" + "http://www.zimbra.com/license/yahoo_public_license_1.1.html" ], - "isOsiApproved": false + "isOsiApproved": false, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/TORQUE-1.1.html", + "reference": "https://spdx.org/licenses/Zed.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/TORQUE-1.1.json", - "referenceNumber": 487, - "name": "TORQUE v2.5+ Software License v1.1", - "licenseId": "TORQUE-1.1", + "detailsUrl": "https://spdx.org/licenses/Zed.json", + "referenceNumber": 152, + "name": "Zed License", + "licenseId": "Zed", "seeAlso": [ - "https://fedoraproject.org/wiki/Licensing/TORQUEv1.1" + "https://fedoraproject.org/wiki/Licensing/Zed" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause.html", + "reference": "https://spdx.org/licenses/Zend-2.0.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause.json", - "referenceNumber": 488, - "name": "BSD 2-Clause \"Simplified\" License", - "licenseId": "BSD-2-Clause", + "detailsUrl": "https://spdx.org/licenses/Zend-2.0.json", + "referenceNumber": 60, + "name": "Zend License v2.0", + "licenseId": "Zend-2.0", "seeAlso": [ - "https://opensource.org/licenses/BSD-2-Clause" + "https://web.archive.org/web/20130517195954/http://www.zend.com/license/2_00.txt" ], - "isOsiApproved": true, + "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/GFDL-1.3-only.html", + "reference": "https://spdx.org/licenses/Zimbra-1.3.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/GFDL-1.3-only.json", - "referenceNumber": 489, - "name": "GNU Free Documentation License v1.3 only", - "licenseId": "GFDL-1.3-only", + "detailsUrl": "https://spdx.org/licenses/Zimbra-1.3.json", + "referenceNumber": 282, + "name": "Zimbra Public License v1.3", + "licenseId": "Zimbra-1.3", "seeAlso": [ - "https://www.gnu.org/licenses/fdl-1.3.txt" + "http://web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html" ], "isOsiApproved": false, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/Artistic-1.0-Perl.html", + "reference": "https://spdx.org/licenses/Zimbra-1.4.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Artistic-1.0-Perl.json", - "referenceNumber": 490, - "name": "Artistic License 1.0 (Perl)", - "licenseId": "Artistic-1.0-Perl", + "detailsUrl": "https://spdx.org/licenses/Zimbra-1.4.json", + "referenceNumber": 424, + "name": "Zimbra Public License v1.4", + "licenseId": "Zimbra-1.4", "seeAlso": [ - "http://dev.perl.org/licenses/artistic.html" + "http://www.zimbra.com/legal/zimbra-public-license-1-4" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/BSD-2-Clause-Views.html", + "reference": "https://spdx.org/licenses/Zlib.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/BSD-2-Clause-Views.json", - "referenceNumber": 491, - "name": "BSD 2-Clause with views sentence", - "licenseId": "BSD-2-Clause-Views", + "detailsUrl": "https://spdx.org/licenses/Zlib.json", + "referenceNumber": 205, + "name": "zlib License", + "licenseId": "Zlib", "seeAlso": [ - "http://www.freebsd.org/copyright/freebsd-license.html", - "https://people.freebsd.org/~ivoras/wine/patch-wine-nvidia.sh", - "https://github.com/protegeproject/protege/blob/master/license.txt" + "http://www.zlib.net/zlib_license.html", + "https://opensource.org/licenses/Zlib" ], - "isOsiApproved": false + "isOsiApproved": true, + "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/NPOSL-3.0.html", + "reference": "https://spdx.org/licenses/zlib-acknowledgement.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/NPOSL-3.0.json", - "referenceNumber": 492, - "name": "Non-Profit Open Software License 3.0", - "licenseId": "NPOSL-3.0", + "detailsUrl": "https://spdx.org/licenses/zlib-acknowledgement.json", + "referenceNumber": 84, + "name": "zlib/libpng License with Acknowledgement", + "licenseId": "zlib-acknowledgement", "seeAlso": [ - "https://opensource.org/licenses/NOSL3.0" + "https://fedoraproject.org/wiki/Licensing/ZlibWithAcknowledgement" ], - "isOsiApproved": true + "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/Minpack.html", + "reference": "https://spdx.org/licenses/ZPL-1.1.html", "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/Minpack.json", - "referenceNumber": 493, - "name": "Minpack License", - "licenseId": "Minpack", + "detailsUrl": "https://spdx.org/licenses/ZPL-1.1.json", + "referenceNumber": 63, + "name": "Zope Public License 1.1", + "licenseId": "ZPL-1.1", "seeAlso": [ - "http://www.netlib.org/minpack/disclaimer", - "https://gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK" + "http://old.zope.org/Resources/License/ZPL-1.1" ], "isOsiApproved": false }, { - "reference": "https://spdx.org/licenses/LGPL-2.1.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-2.1.json", - "referenceNumber": 494, - "name": "GNU Lesser General Public License v2.1 only", - "licenseId": "LGPL-2.1", + "reference": "https://spdx.org/licenses/ZPL-2.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ZPL-2.0.json", + "referenceNumber": 116, + "name": "Zope Public License 2.0", + "licenseId": "ZPL-2.0", "seeAlso": [ - "https://www.gnu.org/licenses/old-licenses/lgpl-2.1-standalone.html", - "https://opensource.org/licenses/LGPL-2.1" + "http://old.zope.org/Resources/License/ZPL-2.0", + "https://opensource.org/licenses/ZPL-2.0" ], "isOsiApproved": true, "isFsfLibre": true }, { - "reference": "https://spdx.org/licenses/LGPL-3.0.html", - "isDeprecatedLicenseId": true, - "detailsUrl": "https://spdx.org/licenses/LGPL-3.0.json", - "referenceNumber": 495, - "name": "GNU Lesser General Public License v3.0 only", - "licenseId": "LGPL-3.0", + "reference": "https://spdx.org/licenses/ZPL-2.1.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/ZPL-2.1.json", + "referenceNumber": 398, + "name": "Zope Public License 2.1", + "licenseId": "ZPL-2.1", "seeAlso": [ - "https://www.gnu.org/licenses/lgpl-3.0-standalone.html", - "https://www.gnu.org/licenses/lgpl+gpl-3.0.txt", - "https://opensource.org/licenses/LGPL-3.0" + "http://old.zope.org/Resources/ZPL/" ], "isOsiApproved": true, "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/CAL-1.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CAL-1.0.json", - "referenceNumber": 496, - "name": "Cryptographic Autonomy License 1.0", - "licenseId": "CAL-1.0", - "seeAlso": [ - "http://cryptographicautonomylicense.com/license-text.html", - "https://opensource.org/licenses/CAL-1.0" - ], - "isOsiApproved": true } ], - "releaseDate": "2022-10-07" + "releaseDate": "2023-01-02" } \ No newline at end of file From efe0a4efcef4a5f75b4439d33bd77106aaef51f9 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 6 Jan 2023 16:58:17 +0000 Subject: [PATCH 048/489] added flags debug print --- src/scanoss/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 67798de0..929156be 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -390,6 +390,8 @@ def scan(parser, args): print_stderr(f'Using Proxy {arg.proxy}...') if args.ca_cert: print_stderr(f'Using Certificate {arg.ca_cert}...') + if flags: + print_stderr(f'Using flags {flags}...') elif not args.quiet: if args.timeout < 5: print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') From 28ee8392d595c436a23788068cbe3fa190ec45f2 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 6 Jan 2023 16:59:36 +0000 Subject: [PATCH 049/489] added support for 503 and general API failures --- .gitignore | 7 +++++++ CHANGELOG.md | 6 ++++++ CONTRIBUTING.md | 2 -- src/scanoss/__init__.py | 2 +- src/scanoss/scanner.py | 3 +++ src/scanoss/scanossapi.py | 6 ++++++ src/scanoss/threadedscanning.py | 37 ++++++++++++++++++++++----------- 7 files changed, 48 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 01f36668..cdeae53c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.wfp *-result.json +*-res.json *.pem .vscode/ gitee_com_* @@ -15,3 +16,9 @@ venv/ .idea src/scanoss/data/build_date.txt bad*.txt +*.csv +*.json + +*.tar +*.tgz +*.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index b80b1afa..956725ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.3] - 2023-01-06 +### Added +- Added support for handling 503 service unavailable responses +- Added latest SPDX license definitions (2.2.7) + ## [1.3.2] - 2022-12-28 ### Added - Added `x-request-id` to all scanning requests @@ -174,3 +179,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.0]: https://github.com/scanoss/scanoss.py/compare/v1.2.3...v1.3.0 [1.3.1]: https://github.com/scanoss/scanoss.py/compare/v1.3.0...v1.3.1 [1.3.2]: https://github.com/scanoss/scanoss.py/compare/v1.3.1...v1.3.2 +[1.3.3]: https://github.com/scanoss/scanoss.py/compare/v1.3.2...v1.3.3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a028d2d..f6113224 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,8 +21,6 @@ Want to submit a pull request? Great! But please follow some basic rules: - If you are changing a source file please make sure that you only include in the changeset the lines changed by you (beware of your editor reformatting the file) - If you are adding functionality, please write a unit test. -When reviewing your pull request, we will follow a checklist similar to this one: https://gist.github.com/audreyr/4feef90445b9680475f2 - ### Licensing The SCANOSS Platform is released under the GPL-2.0 license. If you wish to contribute, you must accept that you are aware of the license under which the project is released, and that your contribution will be released under the same license. Sometimes the GPL-2.0 license is incompatible with other licenses chosen by other projects. Therefore, you must accept that your contribution can also be released under the MIT license, which is the license we choose for those situations. Unless you expressly request otherwise, we may use your name, email address, username or URL for your attribution notice text. The submission of your contribution implies that you agree with these licensing terms. diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index ff44f7f4..6dd9d4f4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.2' +__version__ = '1.3.3' diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 41ab685f..9ee12eea 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -360,6 +360,9 @@ def scan_folder(self, scan_dir: str) -> bool: scan_started = False for root, dirs, files in os.walk(scan_dir): self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') + if self.threaded_scan and self.threaded_scan.stop_scanning(): + self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') + break dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories filtered_files = self.__filter_files(files) # Strip out unwanted files self.print_debug(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}') diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 009b34fe..ab316cae 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -165,6 +165,12 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): else: self.print_stderr(f'Warning: No response received from {self.url}. Retrying...') time.sleep(5) + elif r.status_code == 503: # Service limits have most likely been reached + self.print_stderr(f'ERROR: SCANOSS API rejected the scan request ({request_id}) due to ' + f'service limits being exceeded') + self.print_stderr(f'ERROR: Details: {r.text.strip()}') + raise Exception(f"ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected " + f"for {self.url} due to service limits being exceeded.") elif r.status_code >= 400: if retry > 5: # No response 5 or more times, fail self.save_bad_req_wfp(scan_files, request_id, scan_id) diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index adf1d8fd..d592bdb5 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -66,7 +66,8 @@ def __init__(self, scanapi :ScanossApi, debug: bool = False, trace: bool = False self._bar_count = 0 self._errors = False self._lock = threading.Lock() - self._stop_event = threading.Event() + self._stop_event = threading.Event() # Control when scanning threads should terminate + self._stop_scanning = threading.Event() # Control if the parent process should abort scanning self._threads = [] if nb_threads > MAX_ALLOWED_THREADS: self.print_msg(f'Warning: Requested threads too large: {nb_threads}. Reducing to {MAX_ALLOWED_THREADS}') @@ -133,6 +134,12 @@ def queue_add(self, wfp: str) -> None: def get_queue_size(self) -> int: return self.inputs.qsize() + def stop_scanning(self) -> bool: + """ + Check if we should keep scanning or not + """ + return self._stop_scanning.is_set() + @property def responses(self) -> List[Dict]: """ @@ -186,28 +193,34 @@ def worker_post(self) -> None: """ current_thread = threading.get_ident() self.print_trace(f'Starting worker {current_thread}...') + api_error = False while not self._stop_event.is_set(): wfp = None if not self.inputs.empty(): # Only try to get a message if there is one on the queue try: wfp = self.inputs.get(timeout=5) - self.print_trace(f'Processing input request ({current_thread})...') - count = self.__count_files_in_wfp(wfp) - if wfp is None or wfp == '': - self.print_stderr(f'Warning: Empty WFP in request input: {wfp}') - resp = self.scanapi.scan(wfp, scan_id=current_thread) - if resp: - self.output.put(resp) # Store the output response to later collection - self.update_bar(count) - self.inputs.task_done() - self.print_trace(f'Request complete ({current_thread}).') + if api_error: # API error encountered, so stop processing anymore requests + self.inputs.task_done() # remove request from the queue + else: + self.print_trace(f'Processing input request ({current_thread})...') + count = self.__count_files_in_wfp(wfp) + if wfp is None or wfp == '': + self.print_stderr(f'Warning: Empty WFP in request input: {wfp}') + resp = self.scanapi.scan(wfp, scan_id=current_thread) + if resp: + self.output.put(resp) # Store the output response to later collection + self.update_bar(count) + self.inputs.task_done() + self.print_trace(f'Request complete ({current_thread}).') except queue.Empty as e: self.print_stderr(f'No message available to process ({current_thread}). Checking again...') except Exception as e: - ThreadedScanning.print_stderr(f'ERROR: Problem encountered running scan: {e}') + self.print_stderr(f'ERROR: Problem encountered running scan: {e}. Aborting current thread.') self._errors = True if wfp: self.inputs.task_done() # If there was a WFP being processed, remove it from the queue + api_error = True # Stop processing anymore work requests + self._stop_scanning.set() # Tell the parent process to abort scanning else: time.sleep(1) # Sleep while waiting for the queue depth to build up self.print_trace(f'Thread complete ({current_thread}).') From f7996d87a635b9170650ba48cf3e6f769db1424a Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 6 Jan 2023 19:06:06 +0000 Subject: [PATCH 050/489] fixed docker build warning --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 27305c47..5c2347b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ FROM base as builder RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential gcc \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN mkdir /install WORKDIR /install From 05f71928edb9a4607a2a68918e6c9d63a24596ee Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 12 Jan 2023 09:05:07 +0000 Subject: [PATCH 051/489] project static analysis cleanup --- PACKAGE.md | 2 +- src/scanoss/cli.py | 96 ++++++++++++++--------------- src/scanoss/csvoutput.py | 7 ++- src/scanoss/cyclonedx.py | 3 +- src/scanoss/filecount.py | 3 +- src/scanoss/scancodedeps.py | 4 +- src/scanoss/scanner.py | 105 ++++++++++++++------------------ src/scanoss/scanossgrpc.py | 7 +-- src/scanoss/spdxlite.py | 2 +- src/scanoss/threadedscanning.py | 11 ++-- src/scanoss/winnowing.py | 4 +- 11 files changed, 114 insertions(+), 130 deletions(-) diff --git a/PACKAGE.md b/PACKAGE.md index 85b2c63b..3987ae42 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -60,7 +60,7 @@ From there it is possible to scan a source code folder: #### Scanning for Dependencies The SCANOSS CLI supports dependency decoration. In order for this to work, it requires the installation of scancode: -```python +```bash pip install scancode-toolkit ``` Dependencies can then be decorated by adding the ``--dependencies`` option to the scanner: diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 929156be..df705335 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -27,7 +27,6 @@ import sys from .scanner import Scanner -from .winnowing import Winnowing from .scancodedeps import ScancodeDeps from .scantype import ScanType from .filecount import FileCount @@ -56,7 +55,7 @@ def setup_args() -> None: ) # Sub-command: version p_ver = subparsers.add_parser('version', aliases=['ver'], - description=f'Version of SCANOSS CLI: {__version__}', help='SCANOSS version') + description=f'Version of SCANOSS CLI: {__version__}', help='SCANOSS version') p_ver.set_defaults(func=ver) # Sub-command: scan p_scan = subparsers.add_parser('scan', aliases=['sc'], @@ -70,9 +69,9 @@ def setup_args() -> None: p_scan.add_argument('--dep', '-p', type=str, help='Use a dependency file instead of a folder (optional)' ) - p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file' ) - p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file' ) - p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) + p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file') + p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file') + p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_scan.add_argument('--format', '-f', type=str, choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], help='Result output format (optional - default: plain)' ) @@ -101,7 +100,8 @@ def setup_args() -> None: p_scan.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') - p_scan.add_argument('--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' ) + p_scan.add_argument('--sc-command', type=str, + help='Scancode command and path if required (optional - default scancode).') p_scan.add_argument('--sc-timeout', type=int, default=600, help='Timeout (in seconds) for scancode to complete (optional - default 600)' ) @@ -113,7 +113,7 @@ def setup_args() -> None: p_wfp.set_defaults(func=wfp) p_wfp.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') - p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) + p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') @@ -123,11 +123,11 @@ def setup_args() -> None: help='Scan source code for dependencies, but do not decorate them') p_dep.set_defaults(func=dependency) p_dep.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') - p_dep.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) - p_dep.add_argument('--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' ) + p_dep.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + p_dep.add_argument('--sc-command', type=str, + help='Scancode command and path if required (optional - default scancode).') p_dep.add_argument('--sc-timeout', type=int, default=600, - help='Timeout (in seconds) for scancode to complete (optional - default 600)' - ) + help='Timeout (in seconds) for scancode to complete (optional - default 600)') # Sub-command: file_count p_fc = subparsers.add_parser('file_count', aliases=['fc'], @@ -135,22 +135,20 @@ def setup_args() -> None: help='Search the source tree and produce a file type summary') p_fc.set_defaults(func=file_count) p_fc.add_argument('scan_dir', metavar='DIR', type=str, nargs='?', help='A folder to search') - p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).' ) + p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_fc.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') # Sub-command: convert p_cnv = subparsers.add_parser('convert', aliases=['cv', 'cnv', 'cvrt'], - description=f'Convert results files between formats: {__version__}', - help='Convert file format') + description=f'Convert results files between formats: {__version__}', + help='Convert file format') p_cnv.set_defaults(func=convert) p_cnv.add_argument('--input', '-i', type=str, required=True, help='Input file name') - p_cnv.add_argument('--output','-o', type=str, help='Output result file name (optional - default stdout).' ) - p_cnv.add_argument('--format','-f', type=str, choices=['cyclonedx', 'spdxlite', 'csv'], default='spdxlite', - help='Output format (optional - default: spdxlite)' - ) + p_cnv.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + p_cnv.add_argument('--format', '-f', type=str, choices=['cyclonedx', 'spdxlite', 'csv'], default='spdxlite', + help='Output format (optional - default: spdxlite)') p_cnv.add_argument('--input-format', type=str, choices=['plain'], default='plain', - help='Input format (optional - default: plain)' - ) + help='Input format (optional - default: plain)') # Sub-command: utils p_util = subparsers.add_parser('utils', aliases=['ut', 'util'], @@ -158,8 +156,7 @@ def setup_args() -> None: help='General utility support commands') utils_sub = p_util.add_subparsers(title='Utils Commands', dest='utilsubparser', description='utils sub-commands', - help='utils sub-commands' - ) + help='utils sub-commands') # Utils Sub-command: utils certloc p_c_loc = utils_sub.add_parser('certloc', aliases=['cl'], @@ -169,12 +166,13 @@ def setup_args() -> None: # Utils Sub-command: utils cert-download p_c_dwnld = utils_sub.add_parser('cert-download', aliases=['cdl', 'cert-dl'], - description=f'Download Server SSL Cert: {__version__}', - help='Download the specified server\'s SSL PEM certificate') + description=f'Download Server SSL Cert: {__version__}', + help='Download the specified server\'s SSL PEM certificate') p_c_dwnld.set_defaults(func=utils_cert_download) - p_c_dwnld.add_argument('--hostname', '-n', required=True, type=str, help='Server hostname to download cert from.' ) - p_c_dwnld.add_argument('--port', '-p', required=False, type=int, default=443, help='Server port number (default: 443).' ) - p_c_dwnld.add_argument('--output','-o', type=str, help='Output result file name (optional - default stdout).' ) + p_c_dwnld.add_argument('--hostname', '-n', required=True, type=str, help='Server hostname to download cert from.') + p_c_dwnld.add_argument('--port', '-p', required=False, type=int, default=443, + help='Server port number (default: 443).') + p_c_dwnld.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') # Global command options for p in [p_scan]: @@ -216,18 +214,14 @@ def setup_args() -> None: args.func(parser, args) # Execute the function associated with the sub-command -def ver(parser, args): +def ver(*_): """ Run the "ver" sub-command - Parameters - ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments + :param _: ignored/unused """ print(f'Version: {__version__}') + def file_count(parser, args): """ Run the "file_count" sub-command @@ -259,6 +253,7 @@ def file_count(parser, args): print_stderr(f'Error: Path specified is not a folder: {args.scan_dir}.') exit(1) + def wfp(parser, args): """ Run the "wfp" sub-command @@ -292,6 +287,7 @@ def wfp(parser, args): print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') exit(1) + def get_scan_options(args): """ Parse the scanning options to determine the correct scan settings @@ -387,16 +383,16 @@ def scan(parser, args): if args.obfuscate: print_stderr("Obfuscating file fingerprints...") if args.proxy: - print_stderr(f'Using Proxy {arg.proxy}...') + print_stderr(f'Using Proxy {args.proxy}...') if args.ca_cert: - print_stderr(f'Using Certificate {arg.ca_cert}...') + print_stderr(f'Using Certificate {args.ca_cert}...') if flags: print_stderr(f'Using flags {flags}...') elif not args.quiet: if args.timeout < 5: print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') - if not os.access( os.getcwd(), os.W_OK ): # Make sure the current directory is writable. If not disable saving WFP + if not os.access(os.getcwd(), os.W_OK): # Make sure the current directory is writable. If not disable saving WFP print_stderr(f'Warning: Current directory is not writable: {os.getcwd()}') args.no_wfp_output = True if args.ca_cert and not os.path.exists(args.ca_cert): @@ -438,6 +434,7 @@ def scan(parser, args): print_stderr('No action found to process') exit(1) + def dependency(parser, args): """ Run the "dependency" sub-command @@ -466,6 +463,7 @@ def dependency(parser, args): if not sc_deps.get_dependencies(what_to_scan=args.scan_dir, result_output=scan_output): exit(1) + def convert(parser, args): """ Run the "convert" sub-command @@ -501,28 +499,21 @@ def convert(parser, args): if not success: exit(1) -def utils_certloc(parser, args): + +def utils_certloc(*_): """ Run the "utils certloc" sub-command - Parameters - ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments + :param _: ignored/unused """ import certifi print(f'CA Cert File: {certifi.where()}') -def utils_cert_download(parser, args): + +def utils_cert_download(_, args): """ Run the "utils cert-download" sub-command - Parameters - ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments + :param _: ignore/unused + :param args: Parsed arguments """ import ssl from urllib.parse import urlparse @@ -530,6 +521,8 @@ def utils_cert_download(parser, args): import traceback file = sys.stdout + hostname = 'unset' + port = 'unkown' try: if args.output: file = open(args.output, 'w') @@ -569,6 +562,7 @@ def utils_cert_download(parser, args): print_stderr(f'Saved certificate to {args.output}') file.close() + def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index 6974039e..ba76dc9e 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -146,7 +146,7 @@ def produce_from_file(self, json_file: str, output_file: str = None) -> bool: def produce_from_json(self, data: json, output_file: str = None) -> bool: """ - Produce the CSV output from the input JSON object + Produce the CSV output from the input data :param data: JSON object :param output_file: Output file (optional) :return: True if successful, False otherwise @@ -156,8 +156,9 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: self.print_stderr('ERROR: No CSV data returned for the JSON string provided.') return False # Header row/column details - fields = ['inventory_id', 'path', 'detected_usage', 'detected_component', 'detected_license', 'detected_version', - 'detected_latest', 'detected_purls', 'detected_match', 'detected_lines', 'detected_oss_lines'] + fields = ['inventory_id', 'path', 'detected_usage', 'detected_component', 'detected_license', + 'detected_version', 'detected_latest', 'detected_purls', 'detected_match', 'detected_lines', + 'detected_oss_lines'] file = sys.stdout if not output_file and self.output_file: output_file = self.output_file diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 56e01fb4..1e267bed 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -40,6 +40,7 @@ def __init__(self, debug: bool = False, output_file: str = None): """ Initialise the CycloneDX class """ + super().__init__(debug) self.output_file = output_file self.debug = debug self._spdx = SpdxLite(debug=debug) @@ -163,7 +164,7 @@ def produce_from_file(self, json_file: str, output_file: str = None) -> bool: def produce_from_json(self, data: json, output_file: str = None) -> bool: """ - Produce the CycloneDX output from the input JSON object + Produce the CycloneDX output from the input data :param data: JSON object :param output_file: Output file (optional) :return: True if successful, False otherwise diff --git a/src/scanoss/filecount.py b/src/scanoss/filecount.py index 0d8db090..e3e9d9cc 100644 --- a/src/scanoss/filecount.py +++ b/src/scanoss/filecount.py @@ -91,8 +91,7 @@ def __log_result(self, string, outfile=None): def count_files(self, scan_dir: str) -> bool: """ - Search the specified folder producing counting the file types found. - + Search the specified folder producing counting the file types found :param scan_dir str Directory to scan :return True if successful, False otherwise diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index abd0b94c..e3e3dca7 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -62,7 +62,7 @@ def __log_result(self, string, outfile=None): def remove_interim_file(self, output_file: str = None): """ Remove the temporary Scancode interim file - :param output_file: file to remove (optional) + :param output_file: filename to remove (optional) """ if not output_file and self.output_file: output_file = self.output_file @@ -188,7 +188,7 @@ def get_dependencies(self, output_file: str = None, what_to_scan: str = None, re def run_scan(self, output_file: str = None, what_to_scan: str = None) -> bool: """ Run a scan of the specified file/folder and output the results to temporary file - :param output_file: temporary scancode output file + :param output_file: temporary scancode output filename :param what_to_scan: file/directory to scan :return: True on success, False otherwise """ diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 9ee12eea..68f60de3 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -45,39 +45,36 @@ from . import __version__ FILTERED_DIRS = { # Folders to skip - "nbproject", "nbbuild", "nbdist", "__pycache__", "venv", "_yardoc", "eggs", "wheels", "htmlcov", - "__pypackages__" - } -FILTERED_DIR_EXT = { # Folder endings to skip - ".egg-info" - } + "nbproject", "nbbuild", "nbdist", "__pycache__", "venv", "_yardoc", "eggs", "wheels", "htmlcov", "__pypackages__" +} +FILTERED_DIR_EXT = { # Folder endings to skip + ".egg-info" +} FILTERED_EXT = { # File extensions to skip - ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", - ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake", ".cnf", - ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css", - ".csv", ".dat", ".data", ".doc", ".docx", ".dtd", ".dts", ".iws", ".c9", ".c9revisions", - ".dtsi", ".dump", ".eot", ".eps", ".geojson", ".gdoc", ".gif", - ".glif", ".gmo", ".gradle", ".guess", ".hex", ".htm", ".html", ".ico", ".iml", - ".in", ".inc", ".info", ".ini", ".ipynb", ".jpeg", ".jpg", ".json", ".jsonld", ".lock", - ".log", ".m4", ".map", ".markdown", ".md", ".md5", ".meta", ".mk", ".mxml", - ".o", ".otf", ".out", ".pbtxt", ".pdf", ".pem", ".phtml", ".plist", ".png", - ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", - ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", - ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", - ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".xht", - ".xhtml", ".xls", ".xlsx", ".xml", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", - ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", - ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", - # File endings - "-doc", "changelog", "config", "copying", "license", "authors", "news", - "licenses", "notice", - "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3" - } + ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", + ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake", ".cnf", + ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css", + ".csv", ".dat", ".data", ".doc", ".docx", ".dtd", ".dts", ".iws", ".c9", ".c9revisions", + ".dtsi", ".dump", ".eot", ".eps", ".geojson", ".gdoc", ".gif", + ".glif", ".gmo", ".gradle", ".guess", ".hex", ".htm", ".html", ".ico", ".iml", + ".in", ".inc", ".info", ".ini", ".ipynb", ".jpeg", ".jpg", ".json", ".jsonld", ".lock", + ".log", ".m4", ".map", ".markdown", ".md", ".md5", ".meta", ".mk", ".mxml", + ".o", ".otf", ".out", ".pbtxt", ".pdf", ".pem", ".phtml", ".plist", ".png", + ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", + ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", + ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", + ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".xht", + ".xhtml", ".xls", ".xlsx", ".xml", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", + ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", + ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", + # File endings + "-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", + "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3" +} FILTERED_FILES = { # Files to skip - "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", - "thumbs.db", "babel.config.js", - "license.txt", "license.md", "copying.lib", "makefile" - } + "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", + "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile" +} WFP_FILE_START = "file=" MAX_POST_SIZE = 64 * 1024 # 64k Max post size @@ -168,9 +165,9 @@ def __filter_dirs(self, dirs: list) -> list: dir_list = [] for d in dirs: ignore = False - if d.startswith(".") and not self.hidden_files_folders: # Ignore all . folders unless requested + if d.startswith(".") and not self.hidden_files_folders: # Ignore all . folders unless requested ignore = True - if not ignore and not self.all_folders: # Skip this check if we're allowing all folders + if not ignore and not self.all_folders: # Skip this check if we're allowing all folders d_lower = d.lower() if d_lower in FILTERED_DIRS: # Ignore specific folders ignore = True @@ -225,14 +222,14 @@ def valid_json_file(json_file: str) -> bool: :return bool True if valid, False otherwise """ if not json_file: - self.print_stderr('ERROR: No JSON file provided to parse.') + Scanner.print_stderr('ERROR: No JSON file provided to parse.') return False if not os.path.isfile(json_file): - self.print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') + Scanner.print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') return False try: with open(json_file) as f: - data = json.load(f) + json.load(f) except Exception as e: Scanner.print_stderr(f'Problem parsing JSON file "{json_file}": {e}') return False @@ -256,7 +253,6 @@ def __version_details() -> str: data = f'date: {now.strftime("%Y%m%d%H%M%S")}, utime: {int(now.timestamp())}' return f'tool: scanoss-py, version: {__version__}, {data}' - def __log_result(self, string, outfile=None): """ Logs result to file or STDOUT @@ -372,7 +368,7 @@ def scan_folder(self, scan_dir: str) -> bool: try: f_size = os.stat(path).st_size except Exception as e: - self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink + self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink if f_size > 0: # Ignore broken links and empty files self.print_trace(f'Fingerprinting {path}...') if spinner: @@ -384,7 +380,7 @@ def scan_folder(self, scan_dir: str) -> bool: file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode("utf-8")) - # If the wfp is bigger than the max post size and we already have something stored in the scan block, add it to the queue + # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -395,7 +391,7 @@ def scan_folder(self, scan_dir: str) -> bool: self.threaded_scan.queue_add(scan_block) queue_size += 1 scan_block = '' - if queue_size > self.nb_threads and not scan_started: # Start scanning if we have something to do + if queue_size > self.nb_threads and not scan_started: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( @@ -408,13 +404,12 @@ def scan_folder(self, scan_dir: str) -> bool: spinner.finish() if wfp_list: - if not self.no_wfp_file or not self.threaded_scan: # Write a WFP file if no threading or not not requested + if not self.no_wfp_file or not self.threaded_scan: # Write a WFP file if no threading is requested self.print_debug(f'Writing fingerprints to {self.wfp}') with open(self.wfp, 'w') as f: f.write(''.join(wfp_list)) else: - self.print_debug( f'Skipping writing WFP file {self.wfp}') - wfp_list = None + self.print_debug(f'Skipping writing WFP file {self.wfp}') if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: @@ -423,7 +418,7 @@ def scan_folder(self, scan_dir: str) -> bool: def __run_scan_threaded(self, scan_started: bool, file_count: int) -> bool: """ - Finish scanning the filtered files and but do not wait for it to complete + Start scanning the filtered files but do not wait for it to complete :param scan_started: If the scan has already started or not :param file_count: Number of total files to be scanned :return: True if successful, False otherwise @@ -516,17 +511,16 @@ def __finish_scan_threaded(self, file_map: dict = None) -> bool: else: success = spdxlite.produce_from_str(raw_output) elif self.output_format == 'csv': - csvo = CsvOutput(self.debug, self.scan_output) - if parsed_json: - success = csvo.produce_from_json(parsed_json) - else: - success = csvo.produce_from_str(raw_output) + csvo = CsvOutput(self.debug, self.scan_output) + if parsed_json: + success = csvo.produce_from_json(parsed_json) + else: + success = csvo.produce_from_str(raw_output) else: self.print_stderr(f'ERROR: Unknown output format: {self.output_format}') success = False return success - def scan_file_with_options(self, file: str, file_map: dict = None) -> bool: """ Scan the given file for whatever scaning options that have been configured @@ -555,7 +549,6 @@ def scan_file_with_options(self, file: str, file_map: dict = None) -> bool: success = False return success - def scan_file(self, file: str) -> bool: """ Scan the specified file and produce a result @@ -585,11 +578,8 @@ def scan_file(self, file: str) -> bool: def scan_wfp_file(self, file: str = None) -> bool: """ Scan the contents of the specified WFP file (in the current process) - Parameters - ---------- - file: str - WFP file to scan (optional) - return: True if successful, False otherwise + :param file: Scan the contents of the specified WFP file (in the current process) + :return: True if successful, False otherwise """ success = True wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default @@ -697,7 +687,6 @@ def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> boo if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") cur_size = 0 - scan_size = 0 queue_size = 0 file_count = 0 scan_started = False @@ -818,7 +807,7 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): try: f_size = os.stat(path).st_size except Exception as e: - self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink + self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink if f_size > 0: # Ignore empty files self.print_debug(f'Fingerprinting {path}...') wfps += self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 34d9acd5..7d836063 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -54,7 +54,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu :param debug: :param trace: :param quiet: - :param cert: + :param ca_cert: To set a custom certificate use: GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/certs/cert.pem @@ -104,7 +104,6 @@ def deps_echo(self, message: str = 'Hello there!') -> str: try: metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID - # resp, call = self.dependencies_stub.Echo.with_call(EchoRequest(message=message), metadata=metadata, timeout=3) resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3) except Exception as e: self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' @@ -186,10 +185,8 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st @staticmethod def _load_cert(cert_file: str) -> bytes: - certificate_chain = None with open(cert_file, 'rb') as f: - certificate_chain = f.read() - return certificate_chain + return f.read() # # End of ScanossGrpc Class # diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index f9869f58..21fb0a6b 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -157,7 +157,7 @@ def produce_from_file(self, json_file: str, output_file: str = None) -> bool: def produce_from_json(self, data: json, output_file: str = None) -> bool: """ - Produce the SPDX Lite output from the input JSON object + Produce the SPDX Lite output from the input data :param data: JSON object :param output_file: Output file (optional) :return: True if successful, False otherwise diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index d592bdb5..26fbe27e 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -37,6 +37,7 @@ WFP_FILE_START = "file=" MAX_ALLOWED_THREADS = int(os.environ.get("SCANOSS_MAX_ALLOWED_THREADS")) if os.environ.get("SCANOSS_MAX_ALLOWED_THREADS") else 30 + @dataclass class ThreadedScanning(ScanossBase): """ @@ -48,7 +49,7 @@ class ThreadedScanning(ScanossBase): output: queue.Queue = queue.Queue() bar: Bar = None - def __init__(self, scanapi :ScanossApi, debug: bool = False, trace: bool = False, quiet: bool = False, + def __init__(self, scanapi: ScanossApi, debug: bool = False, trace: bool = False, quiet: bool = False, nb_threads: int = 5 ) -> None: """ @@ -107,6 +108,8 @@ def update_bar(self, amount: int = 0, create: bool = False, file_count: int = 0) """ Update the Progress Bar progress :param amount: amount of progress to update + :param create: create the bar if requested + :param file_count: file count """ try: self._lock.acquire() @@ -212,14 +215,14 @@ def worker_post(self) -> None: self.update_bar(count) self.inputs.task_done() self.print_trace(f'Request complete ({current_thread}).') - except queue.Empty as e: + except queue.Empty: self.print_stderr(f'No message available to process ({current_thread}). Checking again...') except Exception as e: self.print_stderr(f'ERROR: Problem encountered running scan: {e}. Aborting current thread.') self._errors = True if wfp: self.inputs.task_done() # If there was a WFP being processed, remove it from the queue - api_error = True # Stop processing anymore work requests + api_error = True # Stop processing anymore work requests self._stop_scanning.set() # Tell the parent process to abort scanning else: time.sleep(1) # Sleep while waiting for the queue depth to build up @@ -227,4 +230,4 @@ def worker_post(self) -> None: # # End of ThreadedScanning Class -# \ No newline at end of file +# diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 8705dc16..708a5a55 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -130,7 +130,7 @@ def __normalize(byte): Parameters ---------- byte : int - The byte to normalize + The byte to normalise """ if byte < ASCII_0: return 0 @@ -163,7 +163,7 @@ def __skip_snippets(self, file: str, src: str) -> bool: for ending in SKIP_SNIPPET_EXT: if lower_file.endswith(ending): self.print_trace(f'Skipping snippets due to file ending: {file} - {ending}') - return True; + return True src_len = len(src) if src_len == 0 or src_len <= MIN_FILE_SIZE: # Ignore empty or files that are too small self.print_trace(f'Skipping snippets as the file is too small: {file} - {src_len}') From 28f07a9f855d7c048481f4182467d13cd740a4c2 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Mon, 16 Jan 2023 18:53:05 +0000 Subject: [PATCH 052/489] added user-agent header to requests --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- src/scanoss/scanossapi.py | 4 ++++ src/scanoss/scanossgrpc.py | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 956725ac..63c9cf10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.4] - 2023-01-16 +### Added +- Added User-Agent client/version to requests + ## [1.3.3] - 2023-01-06 ### Added - Added support for handling 503 service unavailable responses @@ -180,3 +184,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.1]: https://github.com/scanoss/scanoss.py/compare/v1.3.0...v1.3.1 [1.3.2]: https://github.com/scanoss/scanoss.py/compare/v1.3.1...v1.3.2 [1.3.3]: https://github.com/scanoss/scanoss.py/compare/v1.3.2...v1.3.3 +[1.3.4]: https://github.com/scanoss/scanoss.py/compare/v1.3.3...v1.3.4 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 6dd9d4f4..5c51cda8 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.3' +__version__ = '1.3.4' diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index ab316cae..040eabb1 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -33,6 +33,8 @@ from urllib3.exceptions import InsecureRequestWarning from .scanossbase import ScanossBase +from . import __version__ + DEFAULT_URL = "https://osskb.org/api/scan/direct" # default free service URL DEFAULT_URL2 = "https://scanoss.com/api/scan/direct" # default premium service URL @@ -85,6 +87,8 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st if self.api_key: self.headers['X-Session'] = self.api_key self.headers['x-api-key'] = self.api_key + self.headers['User-Agent'] = f'scanoss-py/{__version__}' + self.headers['user-agent'] = f'scanoss-py/{__version__}' self.sbom = None self.load_sbom() # Load an input SBOM if one is specified if self.trace: diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 7d836063..4cb5771d 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -34,6 +34,7 @@ from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, DependencyResponse from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode from .scanossbase import ScanossBase +from . import __version__ # DEFAULT_URL = "https://osskb.org" DEFAULT_URL = "https://scanoss.com" @@ -72,6 +73,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu self.metadata.append(('x-api-key', api_key)) # Set API key if we have one if ver_details: self.metadata.append(('x-scanoss-client', ver_details)) + self.metadata.append(('user-agent', f'scanoss-py/{__version__}')) secure = True if self.url.startswith('https:') else False # Is it a secure connection? if self.url.startswith('http'): u = urlparse(self.url) From 77fca6de2eaabbf8120b09fa69ec72d5ece6a311 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 31 Jan 2023 17:27:58 +0000 Subject: [PATCH 053/489] make makefile help support numbers --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 60c7a115..ee9340d7 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION=$(shell ./version.py) .PHONY: help help: ## This help - @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) .DEFAULT_GOAL := help From 2ec31fc699790478473e3f278d7bd233448fc59b Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 31 Jan 2023 20:24:58 +0000 Subject: [PATCH 054/489] added extra fields to csv output --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- src/scanoss/csvoutput.py | 15 ++++++++------- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63c9cf10..dd791592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.5] - 2023-01-31 +### Added +- Added extra fields to CSV output (detected_url, detected_path) + ## [1.3.4] - 2023-01-16 ### Added - Added User-Agent client/version to requests @@ -185,3 +189,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.2]: https://github.com/scanoss/scanoss.py/compare/v1.3.1...v1.3.2 [1.3.3]: https://github.com/scanoss/scanoss.py/compare/v1.3.2...v1.3.3 [1.3.4]: https://github.com/scanoss/scanoss.py/compare/v1.3.3...v1.3.4 +[1.3.5]: https://github.com/scanoss/scanoss.py/compare/v1.3.4...v1.3.5 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 5c51cda8..609ac50b 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.4' +__version__ = '1.3.5' diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index ba76dc9e..2f66d643 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -63,8 +63,8 @@ def parse(self, data: json): if not id_details or id_details == 'none': continue matched = d.get("matched", '') - lines = d.get("lines", '') - oss_lines = d.get("oss_lines", '') + lines = d.get("lines", '').replace(',', ';') # swap comma with semicolon to help basic parsers + oss_lines = d.get("oss_lines", '').replace(',', ';') detected = {} if id_details == 'dependency': dependencies = d.get("dependencies") @@ -77,7 +77,7 @@ def parse(self, data: json): self.print_stderr(f'Warning: No PURL found for {f}: {deps}') continue detected['purls'] = purl - for field in ['component', 'version', 'latest']: + for field in ['component', 'version', 'latest', 'url']: detected[field] = deps.get(field, '') licenses = deps.get('licenses') dc = [] @@ -103,7 +103,7 @@ def parse(self, data: json): self.print_stderr(f'Warning: No PURL found for {f}: {file_details}') continue detected['purls'] = ';'.join(pa) - for field in ['component', 'version', 'latest']: + for field in ['component', 'version', 'latest', 'url', 'file']: detected[field] = d.get(field, '') licenses = d.get('licenses') dc = [] @@ -121,7 +121,8 @@ def parse(self, data: json): 'detected_component': detected.get('component'), 'detected_license': detected.get('licenses'), 'detected_version': detected.get('version'), 'detected_latest': detected.get('latest'), - 'detected_purls': detected.get('purls'), + 'detected_purls': detected.get('purls'), 'detected_url': detected.get('url'), + 'detected_path': detected.get('file', ''), 'detected_match': matched, 'detected_lines': lines, 'detected_oss_lines': oss_lines }) row_id = row_id + 1 @@ -157,8 +158,8 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: return False # Header row/column details fields = ['inventory_id', 'path', 'detected_usage', 'detected_component', 'detected_license', - 'detected_version', 'detected_latest', 'detected_purls', 'detected_match', 'detected_lines', - 'detected_oss_lines'] + 'detected_version', 'detected_latest', 'detected_purls', 'detected_url', 'detected_match', + 'detected_lines', 'detected_oss_lines', 'detected_path'] file = sys.stdout if not output_file and self.output_file: output_file = self.output_file From 821924a784715140593a163d1848875225f54c1e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 2 Feb 2023 12:53:32 +0000 Subject: [PATCH 055/489] added pac and grpc_proxy support --- CHANGELOG.md | 6 +++ CLIENT_HELP.md | 28 +++++++++++++- requirements.txt | 1 + setup.py | 2 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 77 +++++++++++++++++++++++++++++++++++++- src/scanoss/scanner.py | 8 ++-- src/scanoss/scanossapi.py | 23 ++++++++++-- src/scanoss/scanossgrpc.py | 47 +++++++++++++++++++---- tests/csvoutput-test.py | 2 +- 10 files changed, 176 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd791592..009ade95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.6] - 2023-02-02 +### Added +- Added support for Proxy Auto-Config (--pac) and GRPC proxy (--grpc-proxy) +- + ## [1.3.5] - 2023-01-31 ### Added - Added extra fields to CSV output (detected_url, detected_path) @@ -190,3 +195,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.3]: https://github.com/scanoss/scanoss.py/compare/v1.3.2...v1.3.3 [1.3.4]: https://github.com/scanoss/scanoss.py/compare/v1.3.3...v1.3.4 [1.3.5]: https://github.com/scanoss/scanoss.py/compare/v1.3.4...v1.3.5 +[1.3.6]: https://github.com/scanoss/scanoss.py/compare/v1.3.5...v1.3.6 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index bdf820e5..5b04d373 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -75,4 +75,30 @@ The REST client support both lowercase & uppercase proxy names, however the gRPC The proxy for REST based calls can also be configured directly on the `scanoss-py` commandline using `--proxy`. For example: ```shell scanoss-py scan --proxy "http://:" -o results.json . -``` \ No newline at end of file +``` +If a separate proxy is required for GRPC calls, please use the `--grpc-proxy` option: +```shell +scanoss-py scan --proxy "http://:" --grpc-proxy "http://:" -D -o results.json . +``` + +### Proxy Auto-Config CLI Options +The `scanoss-py` CLI also supports Proxy Auto-Config (PAC) when scanning using the `--pac` command option. + +It supports three options: +* auto - check the system for a PAC configuration + * `scanoss-py scan --pac auto -o results.json .` +* file - load a local PAC file + * `scanoss-py scan --pac file://proxy.pac -o results.json .` +* url - download a specific PAC file + * `scanoss-py scan --pac https://path.to/proxy.pac -o results.json .` + +### PAC File Evaluation +The `scanoss-py` CLI provides a utility command to help identify if traffic to the SCANOSS services is required over a proxy or not. + +Simply run the following commands find out: +* auto + * `scanoss-py utils pac-proxy --pac auto --url https://osskb.org` +* file + * `scanoss-py utils pac-proxy --pac file://proxy.pac --url https://osskb.org` +* url + * `scanoss-py utils pac-proxy --pac https://path.to/proxy.pac --url https://osskb.org` diff --git a/requirements.txt b/requirements.txt index 99577408..5e48e6fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ binaryornot progress grpcio<=1.42.0 protobuf>=3.16.0,<=3.19.1 +pypac urllib3 \ No newline at end of file diff --git a/setup.py b/setup.py index 3f384344..351d2938 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def get_version(rel_path): long_description_content_type='text/markdown', install_requires=["requests", # TODO Add min req for python 3.10 here - urllib3>=1.26.8 and requests>=2.27.0? "crc32c>=2.2", "binaryornot", "progress", "grpcio<=1.42.0", - "protobuf>=3.16.0,<=3.19.1" + "protobuf>=3.16.0,<=3.19.1", "pypac" ], include_package_data=True, package_data={'': ['data/*.json', 'data/*.txt']}, diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 609ac50b..30ea6c7b 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.5' +__version__ = '1.3.6' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index df705335..0c0d1b1e 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -26,6 +26,8 @@ import os import sys +import pypac + from .scanner import Scanner from .scancodedeps import ScancodeDeps from .scantype import ScanType @@ -174,6 +176,17 @@ def setup_args() -> None: help='Server port number (default: 443).') p_c_dwnld.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + # Utils Sub-command: utils pac-proxy + p_p_proxy = utils_sub.add_parser('pac-proxy', aliases=['pac'], + description=f'Determine Proxy from PAC: {__version__}', + help='Use Proxy Auto-Config to determine proxy configuration') + p_p_proxy.set_defaults(func=utils_pac_proxy) + p_p_proxy.add_argument('--pac', required=False, type=str, default="auto", + help='Proxy auto configuration. Specify a file, http url or "auto" to try to discover it.' + ) + p_p_proxy.add_argument('--url', required=False, type=str, default="https://osskb.org/api", + help='URL to test (default: https://osskb.org/api).') + # Global command options for p in [p_scan]: p.add_argument('--key', '-k', type=str, @@ -189,6 +202,12 @@ def setup_args() -> None: 'Can also use the environment variable "HTTPS_PROXY=:" ' 'and "grcp_proxy=:" for gRPC' ) + p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' + 'Can also use the environment variable "grcp_proxy=:"' + ) + p.add_argument('--pac', type=str, help='Proxy auto configuration (optional). ' + 'Specify a file, http url or "auto" to try to discover it.' + ) p.add_argument('--ca-cert', type=str, help='Alternative certificate PEM file (optional). ' 'Can also use the environment variable ' '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem" and ' @@ -196,7 +215,7 @@ def setup_args() -> None: ) p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') - for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld]: + for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -334,6 +353,10 @@ def scan(parser, args): print_stderr('Please specify a file/folder or fingerprint (--wfp)') parser.parse_args([args.subparser, '-h']) exit(1) + if args.pac and args.proxy: + print_stderr('Please specify one of --proxy or --pac, not both') + parser.parse_args([args.subparser, '-h']) + exit(1) scan_type: str = None sbom_path: str = None if args.identify: @@ -384,6 +407,10 @@ def scan(parser, args): print_stderr("Obfuscating file fingerprints...") if args.proxy: print_stderr(f'Using Proxy {args.proxy}...') + if args.grpc_proxy: + print_stderr(f'Using GRPC Proxy {args.grpc_proxy}...') + if args.pac: + print_stderr(f'Using Proxy Auto-config (PAC) {args.pac}...') if args.ca_cert: print_stderr(f'Using Certificate {args.ca_cert}...') if flags: @@ -398,6 +425,7 @@ def scan(parser, args): if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) + pac_file = get_pac_file(args.pac) scan_options = get_scan_options(args) # Figure out what scanning options we have scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, api_key=args.key, url=args.apiurl, @@ -407,7 +435,8 @@ def scan(parser, args): all_folders=args.all_folders, hidden_files_folders=args.all_hidden, scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, grpc_url=args.api2url, obfuscate=args.obfuscate, - ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, ca_cert=args.ca_cert + ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy, + pac=pac_file, ca_cert=args.ca_cert ) if args.wfp: if not scanner.is_file_or_snippet_scan(): @@ -563,6 +592,50 @@ def utils_cert_download(_, args): file.close() +def utils_pac_proxy(_, args): + """ + Run the "utils pac-proxy" sub-command + :param _: ignore/unused + :param args: Parsed arguments + """ + from pypac.resolver import ProxyResolver + if not args.pac: + print_stderr(f'Error: No pac file option specified.') + exit(1) + pac_file = get_pac_file(args.pac) + if pac_file is None: + print_stderr(f'No proxy configuration for: {args.pac}') + exit(1) + resolver = ProxyResolver(pac_file) + proxies = resolver.get_proxy_for_requests(args.url) + print(f'Proxies: {proxies}\n') + + +def get_pac_file(pac: str): + """ + Get a PAC file if requested. Load the system version (auto), specific local file, or download URL + :param pac: PAC file (auto, file://..., http...) + :return: PAC File object or None + """ + pac_file = None + if pac: + if pac == 'auto': + pac_file = pypac.get_pac() # try to determine the PAC file + elif pac.startswith('file://'): + pac_local = pac.strip('file://') + if not os.path.exists(pac_local): + print_stderr(f'Error: PAC file does not exist: {pac_local}.') + exit(1) + with open(pac_local) as pf: + pac_file = pypac.get_pac(js=pf.read()) + elif pac.startswith('http'): + pac_file = pypac.get_pac(url=pac) + else: + print_stderr(f'Error: Unknown PAC file option: {pac}. Should be one of "auto", "file://", "https://"') + exit(1) + return pac_file + + def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 68f60de3..f8a87da1 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -29,6 +29,7 @@ from progress.bar import Bar from progress.spinner import Spinner +from pypac.parser import PACFile from .scanossapi import ScanossApi from .winnowing import Winnowing @@ -90,7 +91,8 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str post_size: int = 64, timeout: int = 120, no_wfp_file: bool = False, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, - obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, ca_cert: str = None + obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, + ca_cert: str = None, pac: PACFile = None ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning @@ -114,11 +116,11 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, ver_details=ver_details, ignore_cert_errors=ignore_cert_errors, - proxy=proxy, ca_cert=ca_cert + proxy=proxy, ca_cert=ca_cert, pac=pac ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, - ver_details=ver_details, ca_cert=ca_cert + ver_details=ver_details, ca_cert=ca_cert, proxy=proxy, pac=pac, grpc_proxy=grpc_proxy ) self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace) self.nb_threads = nb_threads diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 040eabb1..4c9e4b62 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -31,6 +31,8 @@ import http.client as http_client import urllib3 +from pypac import PACSession +from pypac.parser import PACFile from urllib3.exceptions import InsecureRequestWarning from .scanossbase import ScanossBase from . import __version__ @@ -51,7 +53,7 @@ class ScanossApi(ScanossBase): def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None, url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, timeout: int = 120, ver_details: str = None, ignore_cert_errors: bool = False, - proxy: str = None, ca_cert: str = None): + proxy: str = None, ca_cert: str = None, pac: PACFile = None): """ Initialise the SCANOSS API :param scan_type: Scan type (default identify) @@ -94,14 +96,24 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 + if pac and not proxy: # Setup PAC session if requested (and no proxy has been explicitly set) + self.print_debug(f'Setting up PAC session...') + self.session = PACSession(pac=pac) + else: + self.session = requests.sessions.Session() self.verify = None if self.ignore_cert_errors: self.print_debug(f'Ignoring cert errors...') urllib3.disable_warnings(InsecureRequestWarning) self.verify = False + self.session.verify = False elif ca_cert: self.verify = ca_cert + self.session.cert = ca_cert + self.session.verify = True self.proxies = {'https': proxy, 'http': proxy} if proxy else None + if self. proxies: + self.session.proxies = self.proxies def load_sbom(self): """ @@ -142,9 +154,12 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): retry += 1 try: r = None - r = requests.post(self.url, files=scan_files, data=form_data, headers=self.headers, - timeout=self.timeout, verify=self.verify, proxies=self.proxies - ) + r = self.session.post(self.url, files=scan_files, data=form_data, headers=self.headers, + timeout=self.timeout + ) + # r = requests.post(self.url, files=scan_files, data=form_data, headers=self.headers, + # timeout=self.timeout, verify=self.verify, proxies=self.proxies + # ) except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e: self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 4cb5771d..54a5ed45 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -27,8 +27,11 @@ import grpc import json -from urllib.parse import urlparse + from google.protobuf.json_format import MessageToDict, ParseDict +from pypac.parser import PACFile +from pypac.resolver import ProxyResolver +from urllib.parse import urlparse from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, DependencyResponse @@ -48,7 +51,8 @@ class ScanossGrpc(ScanossBase): """ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - ca_cert: str = None, api_key: str = None, ver_details: str = None): + ca_cert: str = None, api_key: str = None, ver_details: str = None, + proxy: str = None, grpc_proxy: str = None, pac: PACFile = None): """ :param url: @@ -67,7 +71,11 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_GRPC_URL self.url = self.url.lower() + self.orig_url = self.url # Used for proxy lookup self.api_key = api_key if api_key else SCANOSS_API_KEY + self.proxy = proxy + self.grpc_proxy = grpc_proxy + self.pac = pac self.metadata = [] if self.api_key: self.metadata.append(('x-api-key', api_key)) # Set API key if we have one @@ -86,18 +94,25 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu secure = True cert_data = ScanossGrpc._load_cert(ca_cert) self.print_debug(f'Setting up (secure: {secure}) connection to {self.url}...') + self._get_proxy_config() if secure is False: self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) # insecure connection else: if ca_cert is not None: credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate else: - credentials = grpc.ssl_channel_credentials() # secure connection with default certificate + credentials = grpc.ssl_channel_credentials() # secure connection with default certificate self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) + @classmethod + def _load_cert(cls, cert_file: str) -> bytes: + with open(cert_file, 'rb') as f: + return f.read() + def deps_echo(self, message: str = 'Hello there!') -> str: """ Send Echo message to the Dependency service + :param self: :param message: Message to send (default: Hello there!) :return: echo or None """ @@ -185,10 +200,28 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st return False return True - @staticmethod - def _load_cert(cert_file: str) -> bytes: - with open(cert_file, 'rb') as f: - return f.read() + def _get_proxy_config(self): + """ + Set the grpc_proxy/http_proxy/https_proxy environment variables if PAC file has been specified + or if an explicit proxy has been supplied + :param self: + """ + if self.grpc_proxy: + self.print_debug(f'Setting GRPC (grpc_proxy) proxy...') + os.environ["grpc_proxy"] = self.grpc_proxy + elif self.proxy: + self.print_debug(f'Setting GRPC (http_proxy/https_proxy) proxies...') + os.environ["http_proxy"] = self.proxy + os.environ["https_proxy"] = self.proxy + elif self.pac: + self.print_debug(f'Attempting to get GRPC proxy details from PAC for {self.orig_url}...') + resolver = ProxyResolver(self.pac) + proxies = resolver.get_proxy_for_requests(self.orig_url) + if proxies: + self.print_trace(f'Setting proxies: {proxies}') + os.environ["http_proxy"] = proxies.get("http") or "" + os.environ["https_proxy"] = proxies.get("https") or "" + # # End of ScanossGrpc Class # diff --git a/tests/csvoutput-test.py b/tests/csvoutput-test.py index b6c09b69..73e7bcd8 100644 --- a/tests/csvoutput-test.py +++ b/tests/csvoutput-test.py @@ -23,7 +23,7 @@ """ import unittest -from scanoss.csvoutput import CsvOutput +from scanoss.winnowing import Winnowing class MyTestCase(unittest.TestCase): From 806f1d5c71345a5baf1343ac17aae82be27dd870 Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 16:57:21 +0000 Subject: [PATCH 056/489] Create python-publish.yml --- .github/workflows/python-publish.yml | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..5b91340c --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,39 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Build package + run: make dist +# - name: Publish package +# uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 +# with: +# user: __token__ +# password: ${{ secrets.PYPI_API_TOKEN }} From a1e3a4ef9bceffc27b92e39a41e303cf01bd82e2 Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:15:30 +0000 Subject: [PATCH 057/489] Create python-testpypi-publish.yml --- .github/workflows/python-testpypi-publish.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/python-testpypi-publish.yml diff --git a/.github/workflows/python-testpypi-publish.yml b/.github/workflows/python-testpypi-publish.yml new file mode 100644 index 00000000..3938ec40 --- /dev/null +++ b/.github/workflows/python-testpypi-publish.yml @@ -0,0 +1,38 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package to TestPyPI + +on: [workflow_dispatch] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish Test Package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ From 14fd818bdb31684c0bad8c24d7ecbd59a64367cb Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:21:06 +0000 Subject: [PATCH 058/489] changed build commands --- .github/workflows/python-testpypi-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-testpypi-publish.yml b/.github/workflows/python-testpypi-publish.yml index 3938ec40..7fa47707 100644 --- a/.github/workflows/python-testpypi-publish.yml +++ b/.github/workflows/python-testpypi-publish.yml @@ -27,9 +27,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install build + pip install -r requirements-dev.txt - name: Build package - run: python -m build + run: make dist - name: Publish Test Package uses: pypa/gh-action-pypi-publish@release/v1 with: From 5e824a432944794e43081fb71eb8ad65499428f0 Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:26:18 +0000 Subject: [PATCH 059/489] Setting build and deploy commands --- .github/workflows/python-publish.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 5b91340c..4e5a7035 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,11 +1,6 @@ # This workflow will upload a Python Package using Twine when a release is created # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - name: Upload Python Package on: @@ -30,10 +25,10 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - - name: Build package + - name: Build Package run: make dist -# - name: Publish package -# uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 -# with: -# user: __token__ -# password: ${{ secrets.PYPI_API_TOKEN }} + - name: Publish Package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From 65af88950904b6b2df2f6ccb135866f80cd7186f Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:51:16 +0000 Subject: [PATCH 060/489] Create ghcr-publish.yml --- .github/workflows/ghcr-publish.yml | 82 ++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/ghcr-publish.yml diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml new file mode 100644 index 00000000..40492e59 --- /dev/null +++ b/.github/workflows/ghcr-publish.yml @@ -0,0 +1,82 @@ +name: GHCR Container Publish +# Publish a multi-platform container when a package is released + +on: + release: + types: [published] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: scanoss/scanoss-py + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Install the cosign tool except on PR + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + with: + cosign-release: 'v1.11.0' + + # Add support for more platforms with QEMU + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + with: + platforms: "linux/amd64,linux/arm64" + + # Login against a Docker registry except on PR + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: "true" + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} From 303482dece27f06875291390065aafb160ec3c3b Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 18:51:36 +0000 Subject: [PATCH 061/489] Create docker-image-build.yml --- .github/workflows/docker-image-build.yml | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/docker-image-build.yml diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml new file mode 100644 index 00000000..d6bb89ab --- /dev/null +++ b/.github/workflows/docker-image-build.yml @@ -0,0 +1,52 @@ +name: Docker Image Build + +on: [workflow_dispatch] + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Add support for more platforms with QEMU + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + with: + platforms: "linux/amd64,linux/arm64" + + # Login against a Docker registry except on PR + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + From 3f4fd3795ef0d592237670bdaf5f26885e80171e Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 18:55:51 +0000 Subject: [PATCH 062/489] added registry --- .github/workflows/docker-image-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index d6bb89ab..7961c6f0 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -2,6 +2,10 @@ name: Docker Image Build on: [workflow_dispatch] +env: + REGISTRY: ghcr.io + IMAGE_NAME: scanoss/scanoss-py + jobs: build: From 6aa1042f0543c82b723fe50e0b5bf51a20526063 Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 19:00:39 +0000 Subject: [PATCH 063/489] added python build --- .github/workflows/docker-image-build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index 7961c6f0..2c9b8f4f 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -16,6 +16,17 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Build package + run: make dist + # Add support for more platforms with QEMU - name: Set up QEMU uses: docker/setup-qemu-action@v2 From cd624e8e7cf17486bb28bec0678fd57d921bfe25 Mon Sep 17 00:00:00 2001 From: scanoss-cs <77555654+scanoss-cs@users.noreply.github.com> Date: Fri, 3 Feb 2023 19:14:35 +0000 Subject: [PATCH 064/489] added python build --- .github/workflows/ghcr-publish.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 40492e59..5af9aca9 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -1,9 +1,15 @@ name: GHCR Container Publish -# Publish a multi-platform container when a package is released +# Publish a multi-platform container when a package is released (tagged) on: - release: - types: [published] + push: + tags: + - 'v*' + pull_request: + branches: + - 'main' +# release: +# types: [published] env: REGISTRY: ghcr.io @@ -23,7 +29,21 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - + + # Setup and build python package + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Build package + run: make dist + # Install the cosign tool except on PR - name: Install cosign if: github.event_name != 'pull_request' From 2369b770730a55dac14513f6458557a0ee004702 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 09:57:23 +0000 Subject: [PATCH 065/489] workflow cleanup --- .github/workflows/docker-image-build.yml | 6 +++--- .github/workflows/ghcr-publish.yml | 8 +++----- .github/workflows/python-publish.yml | 10 ++++++---- .github/workflows/python-testpypi-publish.yml | 15 ++++++--------- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index 2c9b8f4f..c36ff5b6 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -1,4 +1,5 @@ name: Docker Image Build +# Build a multi-platform docker image on demand (no publish) on: [workflow_dispatch] @@ -7,15 +8,14 @@ env: IMAGE_NAME: scanoss/scanoss-py jobs: - build: - runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3 + # Setup and build the python package - name: Set up Python uses: actions/setup-python@v3 with: @@ -52,7 +52,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # Build and push Docker image with Buildx (don't push on PR) + # Build Docker image with Buildx - name: Build and push Docker image id: build-and-push uses: docker/build-push-action@v4 diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 5af9aca9..6d119013 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -1,4 +1,4 @@ -name: GHCR Container Publish +name: Publish GHCR Container # Publish a multi-platform container when a package is released (tagged) on: @@ -16,14 +16,12 @@ env: IMAGE_NAME: scanoss/scanoss-py jobs: - build: - + deploy: runs-on: ubuntu-latest permissions: contents: read packages: write - # This is used to complete the identity challenge - # with sigstore/fulcio when running outside of PRs. + # This is used to complete the identity challenge with sigstore/fulcio when running outside of PRs. id-token: write steps: diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 4e5a7035..f3c7f2d2 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,7 +1,6 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries +# This workflow will upload a Python Package using Twine to PyPI when a release is created -name: Upload Python Package +name: Publish Python Package - PyPI on: release: @@ -12,21 +11,24 @@ permissions: jobs: deploy: - runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set up Python uses: actions/setup-python@v3 with: python-version: '3.x' + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt + - name: Build Package run: make dist + - name: Publish Package uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/python-testpypi-publish.yml b/.github/workflows/python-testpypi-publish.yml index 7fa47707..e3911b59 100644 --- a/.github/workflows/python-testpypi-publish.yml +++ b/.github/workflows/python-testpypi-publish.yml @@ -1,12 +1,6 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries +# This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package to TestPyPI +name: Publish Python Package to TestPyPI on: [workflow_dispatch] @@ -15,21 +9,24 @@ permissions: jobs: deploy: - runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set up Python uses: actions/setup-python@v3 with: python-version: '3.x' + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt + - name: Build package run: make dist + - name: Publish Test Package uses: pypa/gh-action-pypi-publish@release/v1 with: From 929678e31d45200702d5eb624da58723f5c2729e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 10:56:32 +0000 Subject: [PATCH 066/489] cosign debug --- .github/workflows/ghcr-publish.yml | 18 ++++++++++++++---- .github/workflows/python-publish.yml | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 6d119013..4cb600e7 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -2,6 +2,7 @@ name: Publish GHCR Container # Publish a multi-platform container when a package is released (tagged) on: + workflow_dispatch: push: tags: - 'v*' @@ -45,9 +46,11 @@ jobs: # Install the cosign tool except on PR - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + uses: sigstore/cosign-installer@v2 with: cosign-release: 'v1.11.0' + - name: Check Cosign Version + run: cosign version # Add support for more platforms with QEMU - name: Set up QEMU @@ -86,6 +89,13 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Sign Docker Image + if: ${{ github.event_name != 'pull_request' }} + run: cosign sign --key env://COSIGN_PRIVATE_KEY ${TAGS} + env: + TAGS: ${{ steps.docker_meta.outputs.tags }} + COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} + COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}} # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker @@ -95,6 +105,6 @@ jobs: if: ${{ github.event_name != 'pull_request' }} env: COSIGN_EXPERIMENTAL: "true" - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + TAGS: ${{ steps.docker_meta.outputs.tags }} + # This step uses the identity token to provision an ephemeral certificate against the sigstore community Fulcio instance. + run: cosign sign ${TAGS} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index f3c7f2d2..ad0c58b3 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -3,6 +3,7 @@ name: Publish Python Package - PyPI on: + workflow_dispatch: release: types: [published] From 2e7f5a8b37ed73765f26bf5d6bde12aa2b4b20a4 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 11:27:20 +0000 Subject: [PATCH 067/489] cosign testing --- .github/workflows/ghcr-publish.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 4cb600e7..d234868b 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -90,21 +90,23 @@ jobs: cache-to: type=gha,mode=max - name: Sign Docker Image - if: ${{ github.event_name != 'pull_request' }} - run: cosign sign --key env://COSIGN_PRIVATE_KEY ${TAGS} + continue-on-error: true env: - TAGS: ${{ steps.docker_meta.outputs.tags }} + TAGS: ${{ steps.meta.outputs.tags }} COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}} + if: ${{ github.event_name != 'pull_request' }} + run: cosign sign --key env://COSIGN_PRIVATE_KEY ${TAGS} # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker # repository is public to avoid leaking data. If you would like to publish # transparency data even for private images, pass --force to cosign below. - name: Sign the published Docker image + continue-on-error: true if: ${{ github.event_name != 'pull_request' }} env: COSIGN_EXPERIMENTAL: "true" - TAGS: ${{ steps.docker_meta.outputs.tags }} + TAGS: ${{ steps.meta.outputs.tags }} # This step uses the identity token to provision an ephemeral certificate against the sigstore community Fulcio instance. run: cosign sign ${TAGS} From 88b1d3d8c2d5a7adda44c235d83dc498133f52a2 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 12:08:00 +0000 Subject: [PATCH 068/489] adding metadata --- .github/workflows/ghcr-publish.yml | 27 +++++++++------------------ Dockerfile | 3 +++ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index d234868b..184f9408 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -47,8 +47,7 @@ jobs: - name: Install cosign if: github.event_name != 'pull_request' uses: sigstore/cosign-installer@v2 - with: - cosign-release: 'v1.11.0' + - name: Check Cosign Version run: cosign version @@ -60,7 +59,7 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf with: - platforms: "linux/amd64,linux/arm64" + platforms: "linux/amd64,linux/arm64,linux/arm64/v8,linux/arm64/v7" # Login against a Docker registry except on PR - name: Log into registry ${{ env.REGISTRY }} @@ -90,23 +89,15 @@ jobs: cache-to: type=gha,mode=max - name: Sign Docker Image - continue-on-error: true + if: ${{ github.event_name != 'pull_request' }} env: TAGS: ${{ steps.meta.outputs.tags }} COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}} - if: ${{ github.event_name != 'pull_request' }} - run: cosign sign --key env://COSIGN_PRIVATE_KEY ${TAGS} + run: cosign sign --key env://COSIGN_PRIVATE_KEY --no-tlog-upload=true ${TAGS} - # Sign the resulting Docker image digest except on PRs. - # This will only write to the public Rekor transparency log when the Docker - # repository is public to avoid leaking data. If you would like to publish - # transparency data even for private images, pass --force to cosign below. - - name: Sign the published Docker image - continue-on-error: true - if: ${{ github.event_name != 'pull_request' }} - env: - COSIGN_EXPERIMENTAL: "true" - TAGS: ${{ steps.meta.outputs.tags }} - # This step uses the identity token to provision an ephemeral certificate against the sigstore community Fulcio instance. - run: cosign sign ${TAGS} +# - name: Sign the images with GitHub OIDC Token +# run: cosign sign ${TAGS} +# env: +# TAGS: ${{ steps.meta.outputs.tags }} +# COSIGN_EXPERIMENTAL: true diff --git a/Dockerfile b/Dockerfile index 5c2347b3..2e28cce9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ FROM python:3.10-slim-buster as base LABEL maintainer="SCANOSS " +LABEL org.opencontainers.image.source=https://github.com/scanoss/scanoss.py +LABEL org.opencontainers.image.description="SCANOSS Python CLI Container" +LABEL org.opencontainers.image.licenses=MIT FROM base as builder From 18e6b91e0ee11faf0b75031010354900d43d5db1 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 12:26:06 +0000 Subject: [PATCH 069/489] update platforms --- .github/workflows/ghcr-publish.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 184f9408..04dc2f00 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -59,7 +59,7 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf with: - platforms: "linux/amd64,linux/arm64,linux/arm64/v8,linux/arm64/v7" + platforms: linux/amd64,linux/arm/v7,linux/arm/v8,linux/arm64 # Login against a Docker registry except on PR - name: Log into registry ${{ env.REGISTRY }} @@ -88,13 +88,13 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Sign Docker Image - if: ${{ github.event_name != 'pull_request' }} - env: - TAGS: ${{ steps.meta.outputs.tags }} - COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} - COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}} - run: cosign sign --key env://COSIGN_PRIVATE_KEY --no-tlog-upload=true ${TAGS} +# - name: Sign Docker Image +# if: ${{ github.event_name != 'pull_request' }} +# env: +# TAGS: ${{ steps.meta.outputs.tags }} +# COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}} +# COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}} +# run: cosign sign --key env://COSIGN_PRIVATE_KEY --no-tlog-upload=true ${TAGS} # - name: Sign the images with GitHub OIDC Token # run: cosign sign ${TAGS} From 42b89ec0702023875561a3d37f9538129f6e380e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 12:41:02 +0000 Subject: [PATCH 070/489] fix platform build --- .github/workflows/ghcr-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 04dc2f00..7fa9cb7a 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -56,10 +56,9 @@ jobs: uses: docker/setup-qemu-action@v2 # Workaround: https://github.com/docker/build-push-action/issues/461 +# uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf - name: Setup Docker buildx - uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf - with: - platforms: linux/amd64,linux/arm/v7,linux/arm/v8,linux/arm64 + uses: docker/setup-buildx-action@v2 # Login against a Docker registry except on PR - name: Log into registry ${{ env.REGISTRY }} @@ -85,6 +84,7 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm/v7,linux/arm/v6,linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max From ccdd1fa4862ab15e5434d42ad7a54c6ef02a42ae Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 12:47:23 +0000 Subject: [PATCH 071/489] update platforms --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5e48e6fc..f6ad3cf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ requests crc32c>=2.2 binaryornot progress -grpcio<=1.42.0 -protobuf>=3.16.0,<=3.19.1 +grpcio>1.42.0 +protobuf>3.19.1 pypac urllib3 \ No newline at end of file From 6415869693bc2d38d7d8da3248d1460ffdbb4909 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 4 Feb 2023 12:47:42 +0000 Subject: [PATCH 072/489] update platforms --- .github/workflows/ghcr-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 7fa9cb7a..999475cc 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -84,7 +84,7 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm/v7,linux/arm/v6,linux/arm64 + platforms: linux/amd64,linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max From 181dab994891d2ba442cc1d32c266ed745f47bb6 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 09:09:17 +0000 Subject: [PATCH 073/489] test docker image --- .github/workflows/docker-image-build.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index c36ff5b6..ef3a2acd 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -28,14 +28,11 @@ jobs: run: make dist # Add support for more platforms with QEMU - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 - # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx - uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf - with: - platforms: "linux/amd64,linux/arm64" + uses: docker/setup-buildx-action@v2 # Login against a Docker registry except on PR - name: Log into registry ${{ env.REGISTRY }} @@ -53,7 +50,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} # Build Docker image with Buildx - - name: Build and push Docker image + - name: Build Docker image id: build-and-push uses: docker/build-push-action@v4 with: @@ -64,4 +61,5 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - + - name: Test Image + run: docker run -it ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} From abd3a75c7e629a4b272d18d64eddb56cf55d3518 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 09:17:51 +0000 Subject: [PATCH 074/489] test update --- .github/workflows/ghcr-publish.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/ghcr-publish.yml index 999475cc..e7dd44fa 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/ghcr-publish.yml @@ -88,6 +88,13 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + # Test the docker image + - name: Test Published Image + if: github.event_name != 'pull_request' + run: | + docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + docker run -it ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # - name: Sign Docker Image # if: ${{ github.event_name != 'pull_request' }} # env: From 985db8a3ff84b37541b3e1347c7080edcc504b75 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 09:19:00 +0000 Subject: [PATCH 075/489] test update2 --- .github/workflows/docker-image-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index ef3a2acd..9a786e68 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -62,4 +62,4 @@ jobs: cache-to: type=gha,mode=max - name: Test Image - run: docker run -it ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + run: docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} From 372c9cd28d42870957a88389db6e17fa68f6333d Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 09:45:33 +0000 Subject: [PATCH 076/489] added python test action --- .github/workflows/docker-image-build.yml | 40 +++++++++++++----------- .github/workflows/python-test.yml | 39 +++++++++++++++++++++++ 2 files changed, 60 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/python-test.yml diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index 9a786e68..f2a13fee 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -4,7 +4,6 @@ name: Docker Image Build on: [workflow_dispatch] env: - REGISTRY: ghcr.io IMAGE_NAME: scanoss/scanoss-py jobs: @@ -35,19 +34,19 @@ jobs: uses: docker/setup-buildx-action@v2 # Login against a Docker registry except on PR - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Extract metadata (tags, labels) for Docker - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} +# - name: Log into registry ${{ env.REGISTRY }} +# uses: docker/login-action@v2 +# with: +# registry: ${{ env.REGISTRY }} +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} +# +# # Extract metadata (tags, labels) for Docker +# - name: Extract Docker metadata +# id: meta +# uses: docker/metadata-action@v4 +# with: +# images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} # Build Docker image with Buildx - name: Build Docker image @@ -56,10 +55,13 @@ jobs: with: context: . push: false - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + tags: ${{ env.IMAGE_NAME }}:latest + outputs: type=docker,dest=/tmp/scanoss-py.tar - name: Test Image - run: docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + run: | + docker load --input /tmp/scanoss-py.tar + docker image ls -a + docker run ${{ env.IMAGE_NAME }} version + docker run ${{ env.IMAGE_NAME }} help + docker run ${{ env.IMAGE_NAME }} scan -o results.json tests diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml new file mode 100644 index 00000000..08ba56da --- /dev/null +++ b/.github/workflows/python-test.yml @@ -0,0 +1,39 @@ +# This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) + +name: Publish Python Package to TestPyPI + +on: [workflow_dispatch] + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Build Local Package + run: make dist + + - name: Test Package + run: | + pip install -r requirements.txt + pip install dist/scanoss-*-py3-none-any.whl + which scanoss-py + scanoss-py version + scanoss-py + scanoss-py scan src > result.json + echo "ID Count: " + cat results.json | grep '"id":' | wc -l From bdd9757200d14979068a10ae32ddcef34a6cc134 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 10:02:14 +0000 Subject: [PATCH 077/489] package testing --- .github/workflows/docker-image-build.yml | 12 ++++++------ .github/workflows/python-test.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index f2a13fee..4afb00cb 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -11,15 +11,15 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository + - name: Checkout Repository uses: actions/checkout@v3 # Setup and build the python package - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.x' - - name: Install dependencies + python-version: '3.10.x' + - name: Install Dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt @@ -49,7 +49,7 @@ jobs: # images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} # Build Docker image with Buildx - - name: Build Docker image + - name: Build Docker Image id: build-and-push uses: docker/build-push-action@v4 with: @@ -58,10 +58,10 @@ jobs: tags: ${{ env.IMAGE_NAME }}:latest outputs: type=docker,dest=/tmp/scanoss-py.tar - - name: Test Image + - name: Test Docker Image run: | docker load --input /tmp/scanoss-py.tar docker image ls -a docker run ${{ env.IMAGE_NAME }} version - docker run ${{ env.IMAGE_NAME }} help docker run ${{ env.IMAGE_NAME }} scan -o results.json tests + cat results.json | grep '"id":' | wc -l \ No newline at end of file diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 08ba56da..d89f08ba 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -1,6 +1,6 @@ # This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) -name: Publish Python Package to TestPyPI +name: Build/Test Local Python Package on: [workflow_dispatch] @@ -8,7 +8,7 @@ permissions: contents: read jobs: - deploy: + build: runs-on: ubuntu-latest steps: @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.x' + python-version: '3.10.x' - name: Install Dependencies run: | From 75db9c6399abd27b5e66da796ce439be6d729742 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 10:09:21 +0000 Subject: [PATCH 078/489] updated commands --- .github/workflows/python-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index d89f08ba..503a9e4f 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -33,7 +33,6 @@ jobs: pip install dist/scanoss-*-py3-none-any.whl which scanoss-py scanoss-py version - scanoss-py scanoss-py scan src > result.json echo "ID Count: " cat results.json | grep '"id":' | wc -l From 8f738a85b8c46c2a1a9e46115bfed0748e0e0368 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 10:15:05 +0000 Subject: [PATCH 079/489] split testing commands --- .github/workflows/python-test.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 503a9e4f..6eda4c16 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -27,12 +27,16 @@ jobs: - name: Build Local Package run: make dist - - name: Test Package + - name: Install Test Package run: | pip install -r requirements.txt pip install dist/scanoss-*-py3-none-any.whl which scanoss-py + + - name: Run Tests + run: | + which scanoss-py scanoss-py version - scanoss-py scan src > result.json + scanoss-py scan src > results.json echo "ID Count: " cat results.json | grep '"id":' | wc -l From ac70bef3c2ff07a3553389eba130045a930202b4 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 10:21:19 +0000 Subject: [PATCH 080/489] updated test commands --- .github/workflows/docker-image-build.yml | 2 +- .github/workflows/python-test.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index 4afb00cb..5cfbfbfc 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -64,4 +64,4 @@ jobs: docker image ls -a docker run ${{ env.IMAGE_NAME }} version docker run ${{ env.IMAGE_NAME }} scan -o results.json tests - cat results.json | grep '"id":' | wc -l \ No newline at end of file + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" \ No newline at end of file diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 6eda4c16..bf1e4c3c 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -38,5 +38,4 @@ jobs: which scanoss-py scanoss-py version scanoss-py scan src > results.json - echo "ID Count: " - cat results.json | grep '"id":' | wc -l + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" From c16dc435867e92af96315fe75c9651486c40a01a Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 10:35:48 +0000 Subject: [PATCH 081/489] fix test commands --- .github/workflows/docker-image-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/docker-image-build.yml index 5cfbfbfc..9009368b 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/docker-image-build.yml @@ -63,5 +63,5 @@ jobs: docker load --input /tmp/scanoss-py.tar docker image ls -a docker run ${{ env.IMAGE_NAME }} version - docker run ${{ env.IMAGE_NAME }} scan -o results.json tests + docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" \ No newline at end of file From 09d546b8ac73463a5b66f5e259be761e8917b19c Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 12:05:17 +0000 Subject: [PATCH 082/489] updating workflow names --- ...age-build.yml => container-local-test.yml} | 40 ++++------ ...publish.yml => container-publish-ghcr.yml} | 29 ++++---- ...{python-test.yml => python-local-test.yml} | 14 +++- .github/workflows/python-publish-pypi.yml | 73 +++++++++++++++++++ .github/workflows/python-publish-testpypi.yml | 72 ++++++++++++++++++ .github/workflows/python-publish.yml | 37 ---------- .github/workflows/python-testpypi-publish.yml | 35 --------- 7 files changed, 184 insertions(+), 116 deletions(-) rename .github/workflows/{docker-image-build.yml => container-local-test.yml} (62%) rename .github/workflows/{ghcr-publish.yml => container-publish-ghcr.yml} (83%) rename .github/workflows/{python-test.yml => python-local-test.yml} (84%) create mode 100644 .github/workflows/python-publish-pypi.yml create mode 100644 .github/workflows/python-publish-testpypi.yml delete mode 100644 .github/workflows/python-publish.yml delete mode 100644 .github/workflows/python-testpypi-publish.yml diff --git a/.github/workflows/docker-image-build.yml b/.github/workflows/container-local-test.yml similarity index 62% rename from .github/workflows/docker-image-build.yml rename to .github/workflows/container-local-test.yml index 9009368b..6b8c24a9 100644 --- a/.github/workflows/docker-image-build.yml +++ b/.github/workflows/container-local-test.yml @@ -1,7 +1,14 @@ -name: Docker Image Build -# Build a multi-platform docker image on demand (no publish) - -on: [workflow_dispatch] +name: Build/Test Local Container +# Build a docker image on demand and run a local test (connecting to osskb.org) + +on: + workflow_dispatch: +# push: +# branches: +# - 'main' +# pull_request: +# branches: +# - 'main' env: IMAGE_NAME: scanoss/scanoss-py @@ -19,35 +26,18 @@ jobs: uses: actions/setup-python@v3 with: python-version: '3.10.x' + - name: Install Dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - - name: Build package - run: make dist - # Add support for more platforms with QEMU -# - name: Set up QEMU -# uses: docker/setup-qemu-action@v2 + - name: Build Package + run: make dist - name: Setup Docker buildx uses: docker/setup-buildx-action@v2 - # Login against a Docker registry except on PR -# - name: Log into registry ${{ env.REGISTRY }} -# uses: docker/login-action@v2 -# with: -# registry: ${{ env.REGISTRY }} -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} -# -# # Extract metadata (tags, labels) for Docker -# - name: Extract Docker metadata -# id: meta -# uses: docker/metadata-action@v4 -# with: -# images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # Build Docker image with Buildx - name: Build Docker Image id: build-and-push @@ -64,4 +54,4 @@ jobs: docker image ls -a docker run ${{ env.IMAGE_NAME }} version docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" \ No newline at end of file + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" diff --git a/.github/workflows/ghcr-publish.yml b/.github/workflows/container-publish-ghcr.yml similarity index 83% rename from .github/workflows/ghcr-publish.yml rename to .github/workflows/container-publish-ghcr.yml index e7dd44fa..177b1bf5 100644 --- a/.github/workflows/ghcr-publish.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -1,14 +1,11 @@ name: Publish GHCR Container -# Publish a multi-platform container when a package is released (tagged) +# Publish a multi-platform container when a version is tagged on: workflow_dispatch: push: tags: - - 'v*' - pull_request: - branches: - - 'main' + - 'v*.*.*' # release: # types: [published] @@ -26,16 +23,16 @@ jobs: id-token: write steps: - - name: Checkout repository + - name: Checkout Repository uses: actions/checkout@v3 # Setup and build python package - name: Set up Python uses: actions/setup-python@v3 with: - python-version: '3.x' + python-version: '3.10.x' - - name: Install dependencies + - name: Install Dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt @@ -44,12 +41,12 @@ jobs: run: make dist # Install the cosign tool except on PR - - name: Install cosign - if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@v2 - - - name: Check Cosign Version - run: cosign version +# - name: Install cosign +# if: github.event_name != 'pull_request' +# uses: sigstore/cosign-installer@v2 +# +# - name: Check Cosign Version +# run: cosign version # Add support for more platforms with QEMU - name: Set up QEMU @@ -93,7 +90,9 @@ jobs: if: github.event_name != 'pull_request' run: | docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - docker run -it ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} version + docker run ${{ env.IMAGE_NAME }} scan -o results.json tests + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" # - name: Sign Docker Image # if: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-local-test.yml similarity index 84% rename from .github/workflows/python-test.yml rename to .github/workflows/python-local-test.yml index bf1e4c3c..e431f5d4 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-local-test.yml @@ -1,8 +1,14 @@ -# This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) - name: Build/Test Local Python Package +# This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) -on: [workflow_dispatch] +on: + workflow_dispatch: +# push: +# branches: +# - 'main' +# pull_request: +# branches: +# - 'main' permissions: contents: read @@ -37,5 +43,5 @@ jobs: run: | which scanoss-py scanoss-py version - scanoss-py scan src > results.json + scanoss-py scan tests > results.json echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml new file mode 100644 index 00000000..8f67c5e7 --- /dev/null +++ b/.github/workflows/python-publish-pypi.yml @@ -0,0 +1,73 @@ +name: Publish Python Package - PyPI +# This workflow will upload a Python Package using Twine to PyPI when a release is created + +on: + workflow_dispatch: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Build Package + run: make dist + + - name: Install Test Package + run: | + pip install -r requirements.txt + pip install dist/scanoss-*-py3-none-any.whl + which scanoss-py + + - name: Run Local Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + pip uninstall scanoss + + - name: Publish Package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' + + - name: Install Remote Package + run: | + pip install --upgrade scanoss + which scanoss-py + + - name: Run Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml new file mode 100644 index 00000000..759a05e8 --- /dev/null +++ b/.github/workflows/python-publish-testpypi.yml @@ -0,0 +1,72 @@ +name: Publish Python Package - TestPyPI +# This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) + +on: [workflow_dispatch] + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Build Package + run: make dist + + - name: Install Test Package + run: | + pip install -r requirements.txt + pip install dist/scanoss-*-py3-none-any.whl + which scanoss-py + + - name: Run Local Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + pip uninstall scanoss + + - name: Publish Test Package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' + + - name: Install Remote Package + run: | + pip install -r requirements.txt + pip install -i https://test.pypi.org/simple/ --upgrade scanoss + which scanoss-py + + - name: Run Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index ad0c58b3..00000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This workflow will upload a Python Package using Twine to PyPI when a release is created - -name: Publish Python Package - PyPI - -on: - workflow_dispatch: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Build Package - run: make dist - - - name: Publish Package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/python-testpypi-publish.yml b/.github/workflows/python-testpypi-publish.yml deleted file mode 100644 index e3911b59..00000000 --- a/.github/workflows/python-testpypi-publish.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) - -name: Publish Python Package to TestPyPI - -on: [workflow_dispatch] - -permissions: - contents: read - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Build package - run: make dist - - - name: Publish Test Package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ From 444d00388526777a1c23313f080b122b25685a58 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 12:41:20 +0000 Subject: [PATCH 083/489] configured triggers --- .github/workflows/container-local-test.yml | 12 ++++++------ .github/workflows/python-local-test.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 6b8c24a9..25270ab4 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -3,12 +3,12 @@ name: Build/Test Local Container on: workflow_dispatch: -# push: -# branches: -# - 'main' -# pull_request: -# branches: -# - 'main' + push: + branches: + - 'main' + pull_request: + branches: + - 'main' env: IMAGE_NAME: scanoss/scanoss-py diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index e431f5d4..0d5f3880 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -3,12 +3,12 @@ name: Build/Test Local Python Package on: workflow_dispatch: -# push: -# branches: -# - 'main' -# pull_request: -# branches: -# - 'main' + push: + branches: + - 'main' + pull_request: + branches: + - 'main' permissions: contents: read From 48140da34b9f340b57c72f4450a969b22b55e22e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 14:37:35 +0000 Subject: [PATCH 084/489] added conditional jobs --- .github/workflows/container-local-test.yml | 7 +- .github/workflows/python-local-test.yml | 59 ++++++----- .github/workflows/python-publish-pypi.yml | 96 +++++++++-------- .github/workflows/python-publish-testpypi.yml | 100 ++++++++++-------- 4 files changed, 144 insertions(+), 118 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 25270ab4..525c7801 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -54,4 +54,9 @@ jobs: docker image ls -a docker run ${{ env.IMAGE_NAME }} version docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 0d5f3880..98c4f71a 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -18,30 +18,35 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.10.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Build Local Package - run: make dist - - - name: Install Test Package - run: | - pip install -r requirements.txt - pip install dist/scanoss-*-py3-none-any.whl - which scanoss-py - - - name: Run Tests - run: | - which scanoss-py - scanoss-py version - scanoss-py scan tests > results.json - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Build Local Package + run: make dist + + - name: Install Test Package + run: | + pip install -r requirements.txt + pip install dist/scanoss-*-py3-none-any.whl + which scanoss-py + + - name: Run Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 8f67c5e7..d482d440 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -4,7 +4,7 @@ name: Publish Python Package - PyPI on: workflow_dispatch: release: - types: [published] + types: [ published ] permissions: contents: read @@ -14,60 +14,68 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.10.x' + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt - - name: Build Package - run: make dist + - name: Build Package + run: make dist - - name: Install Test Package - run: | - pip install -r requirements.txt - pip install dist/scanoss-*-py3-none-any.whl - which scanoss-py + - name: Install Test Package + run: | + pip install -r requirements.txt + pip install dist/scanoss-*-py3-none-any.whl + which scanoss-py - - name: Run Local Tests - run: | - which scanoss-py - scanoss-py version - scanoss-py scan tests > results.json - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" - pip uninstall scanoss + - name: Run Local Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + pip uninstall -y scanoss - - name: Publish Package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + - name: Publish Package + id: publish-pypi-package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} test: + if: success() + needs: [ publish-pypi-package ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.10.x' + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' - - name: Install Remote Package - run: | - pip install --upgrade scanoss - which scanoss-py + - name: Install Remote Package + run: | + pip install --upgrade scanoss + which scanoss-py - - name: Run Tests - run: | - which scanoss-py - scanoss-py version - scanoss-py scan tests > results.json - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + - name: Run Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 759a05e8..318218c4 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -1,7 +1,7 @@ name: Publish Python Package - TestPyPI # This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) -on: [workflow_dispatch] +on: [ workflow_dispatch ] permissions: contents: read @@ -11,62 +11,70 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.10.x' + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt - - name: Build Package - run: make dist + - name: Build Package + run: make dist - - name: Install Test Package - run: | - pip install -r requirements.txt - pip install dist/scanoss-*-py3-none-any.whl - which scanoss-py + - name: Install Test Package + run: | + pip install -r requirements.txt + pip install dist/scanoss-*-py3-none-any.whl + which scanoss-py - - name: Run Local Tests - run: | - which scanoss-py - scanoss-py version - scanoss-py scan tests > results.json - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" - pip uninstall scanoss + - name: Run Local Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + pip uninstall -y scanoss - - name: Publish Test Package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ + - name: Publish Test Package + id: publish-testpypi-package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ test: + if: success() + needs: [ publish-testpypi-package ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.10.x' + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' - - name: Install Remote Package - run: | - pip install -r requirements.txt - pip install -i https://test.pypi.org/simple/ --upgrade scanoss - which scanoss-py + - name: Install Remote Package + run: | + pip install -r requirements.txt + pip install -i https://test.pypi.org/simple/ --upgrade scanoss + which scanoss-py - - name: Run Tests - run: | - which scanoss-py - scanoss-py version - scanoss-py scan tests > results.json - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + - name: Run Tests + run: | + which scanoss-py + scanoss-py version + scanoss-py scan tests > results.json + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi From d346dddac167390d0877307fca1b98df450823c9 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 14:45:02 +0000 Subject: [PATCH 085/489] fixed job dependency --- .github/workflows/container-publish-ghcr.yml | 31 +++++++++++-------- .github/workflows/python-publish-pypi.yml | 2 +- .github/workflows/python-publish-testpypi.yml | 2 +- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 177b1bf5..7f25a61e 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -25,35 +25,27 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v3 - + # Setup and build python package - name: Set up Python uses: actions/setup-python@v3 with: python-version: '3.10.x' - + - name: Install Dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - + - name: Build package run: make dist - - # Install the cosign tool except on PR -# - name: Install cosign -# if: github.event_name != 'pull_request' -# uses: sigstore/cosign-installer@v2 -# -# - name: Check Cosign Version -# run: cosign version # Add support for more platforms with QEMU - name: Set up QEMU uses: docker/setup-qemu-action@v2 # Workaround: https://github.com/docker/build-push-action/issues/461 -# uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + # uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf - name: Setup Docker buildx uses: docker/setup-buildx-action@v2 @@ -92,8 +84,21 @@ jobs: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} version docker run ${{ env.IMAGE_NAME }} scan -o results.json tests - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi +# Install the cosign tool except on PR +# - name: Install cosign +# if: github.event_name != 'pull_request' +# uses: sigstore/cosign-installer@v2 +# +# - name: Check Cosign Version +# run: cosign version +# # - name: Sign Docker Image # if: ${{ github.event_name != 'pull_request' }} # env: diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index d482d440..f5d548bf 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -52,7 +52,7 @@ jobs: test: if: success() - needs: [ publish-pypi-package ] + needs: [ deploy ] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 318218c4..7609ea1d 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -50,7 +50,7 @@ jobs: test: if: success() - needs: [ publish-testpypi-package ] + needs: [ deploy ] runs-on: ubuntu-latest steps: From 8e0f59a029e305f562184d10e15934a3e6a8bfc9 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 15:02:35 +0000 Subject: [PATCH 086/489] added support for latest protobuf --- .github/workflows/container-publish-ghcr.yml | 2 +- .github/workflows/python-publish-pypi.yml | 1 - .github/workflows/python-publish-testpypi.yml | 1 - .gitignore | 3 +- CHANGELOG.md | 7 +- setup.py | 4 +- src/scanoss/__init__.py | 2 +- .../api/common/v2/scanoss_common_pb2.py | 207 +------ .../components/v2/scanoss_components_pb2.py | 537 +----------------- .../v2/scanoss_dependencies_pb2.py | 471 +-------------- .../api/scanning/v2/scanoss_scanning_pb2.py | 52 +- .../v2/scanoss_vulnerabilities_pb2.py | 460 +-------------- 12 files changed, 118 insertions(+), 1629 deletions(-) diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 7f25a61e..e9c570ce 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -83,7 +83,7 @@ jobs: run: | docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} version - docker run ${{ env.IMAGE_NAME }} scan -o results.json tests + docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index f5d548bf..b1e53e2d 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -44,7 +44,6 @@ jobs: pip uninstall -y scanoss - name: Publish Package - id: publish-pypi-package uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 7609ea1d..3f8bf518 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -41,7 +41,6 @@ jobs: pip uninstall -y scanoss - name: Publish Test Package - id: publish-testpypi-package uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ diff --git a/.gitignore b/.gitignore index cdeae53c..8968fc1c 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,8 @@ src/scanoss/data/build_date.txt bad*.txt *.csv *.json - *.tar *.tgz *.gz +*.zip +local-*.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 009ade95..dc113c0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.3.7] - 2023-02-07 +### Added +- Upgrade to the latest protobuf and grpcio packages +- Added GH Actions for building + ## [1.3.6] - 2023-02-02 ### Added - Added support for Proxy Auto-Config (--pac) and GRPC proxy (--grpc-proxy) -- ## [1.3.5] - 2023-01-31 ### Added @@ -196,3 +200,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.4]: https://github.com/scanoss/scanoss.py/compare/v1.3.3...v1.3.4 [1.3.5]: https://github.com/scanoss/scanoss.py/compare/v1.3.4...v1.3.5 [1.3.6]: https://github.com/scanoss/scanoss.py/compare/v1.3.5...v1.3.6 +[1.3.7]: https://github.com/scanoss/scanoss.py/compare/v1.3.6...v1.3.7 diff --git a/setup.py b/setup.py index 351d2938..e3070024 100644 --- a/setup.py +++ b/setup.py @@ -28,8 +28,8 @@ def get_version(rel_path): long_description=read("PACKAGE.md"), long_description_content_type='text/markdown', install_requires=["requests", # TODO Add min req for python 3.10 here - urllib3>=1.26.8 and requests>=2.27.0? - "crc32c>=2.2", "binaryornot", "progress", "grpcio<=1.42.0", - "protobuf>=3.16.0,<=3.19.1", "pypac" + "crc32c>=2.2", "binaryornot", "progress", "grpcio>1.42.0", + "protobuf>3.19.1", "pypac" ], include_package_data=True, package_data={'': ['data/*.json', 'data/*.txt']}, diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 30ea6c7b..b284dcf7 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.6' +__version__ = '1.3.7' diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index c787428e..56fee94b 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -2,10 +2,9 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/common/v2/scanoss-common.proto """Generated protocol buffer code.""" -from google.protobuf.internal import enum_type_wrapper +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -14,194 +13,20 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='scanoss/api/common/v2/scanoss-common.proto', - package='scanoss.api.common.v2', - syntax='proto3', - serialized_options=b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3' -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') -_STATUSCODE = _descriptor.EnumDescriptor( - name='StatusCode', - full_name='scanoss.api.common.v2.StatusCode', - filename=None, - file=DESCRIPTOR, - create_key=_descriptor._internal_create_key, - values=[ - _descriptor.EnumValueDescriptor( - name='UNSPECIFIED', index=0, number=0, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='SUCCESS', index=1, number=1, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='SUCCEEDED_WITH_WARNINGS', index=2, number=2, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='WARNING', index=3, number=3, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - _descriptor.EnumValueDescriptor( - name='FAILED', index=4, number=4, - serialized_options=None, - type=None, - create_key=_descriptor._internal_create_key), - ], - containing_type=None, - serialized_options=None, - serialized_start=220, - serialized_end=316, -) -_sym_db.RegisterEnumDescriptor(_STATUSCODE) +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: -StatusCode = enum_type_wrapper.EnumTypeWrapper(_STATUSCODE) -UNSPECIFIED = 0 -SUCCESS = 1 -SUCCEEDED_WITH_WARNINGS = 2 -WARNING = 3 -FAILED = 4 - - - -_STATUSRESPONSE = _descriptor.Descriptor( - name='StatusResponse', - full_name='scanoss.api.common.v2.StatusResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='status', full_name='scanoss.api.common.v2.StatusResponse.status', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='message', full_name='scanoss.api.common.v2.StatusResponse.message', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=69, - serialized_end=153, -) - - -_ECHOREQUEST = _descriptor.Descriptor( - name='EchoRequest', - full_name='scanoss.api.common.v2.EchoRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='message', full_name='scanoss.api.common.v2.EchoRequest.message', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=155, - serialized_end=185, -) - - -_ECHORESPONSE = _descriptor.Descriptor( - name='EchoResponse', - full_name='scanoss.api.common.v2.EchoResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='message', full_name='scanoss.api.common.v2.EchoResponse.message', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=187, - serialized_end=218, -) - -_STATUSRESPONSE.fields_by_name['status'].enum_type = _STATUSCODE -DESCRIPTOR.message_types_by_name['StatusResponse'] = _STATUSRESPONSE -DESCRIPTOR.message_types_by_name['EchoRequest'] = _ECHOREQUEST -DESCRIPTOR.message_types_by_name['EchoResponse'] = _ECHORESPONSE -DESCRIPTOR.enum_types_by_name['StatusCode'] = _STATUSCODE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -StatusResponse = _reflection.GeneratedProtocolMessageType('StatusResponse', (_message.Message,), { - 'DESCRIPTOR' : _STATUSRESPONSE, - '__module__' : 'scanoss.api.common.v2.scanoss_common_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.common.v2.StatusResponse) - }) -_sym_db.RegisterMessage(StatusResponse) - -EchoRequest = _reflection.GeneratedProtocolMessageType('EchoRequest', (_message.Message,), { - 'DESCRIPTOR' : _ECHOREQUEST, - '__module__' : 'scanoss.api.common.v2.scanoss_common_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.common.v2.EchoRequest) - }) -_sym_db.RegisterMessage(EchoRequest) - -EchoResponse = _reflection.GeneratedProtocolMessageType('EchoResponse', (_message.Message,), { - 'DESCRIPTOR' : _ECHORESPONSE, - '__module__' : 'scanoss.api.common.v2.scanoss_common_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.common.v2.EchoResponse) - }) -_sym_db.RegisterMessage(EchoResponse) - - -DESCRIPTOR._options = None + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' + _STATUSCODE._serialized_start=220 + _STATUSCODE._serialized_end=316 + _STATUSRESPONSE._serialized_start=69 + _STATUSRESPONSE._serialized_end=153 + _ECHOREQUEST._serialized_start=155 + _ECHOREQUEST._serialized_end=185 + _ECHORESPONSE._serialized_start=187 + _ECHORESPONSE._serialized_end=218 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index b4b09a6b..8ae503da 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -2,9 +2,9 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/components/v2/scanoss-components.proto """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -14,511 +14,30 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor.FileDescriptor( - name='scanoss/api/components/v2/scanoss-components.proto', - package='scanoss.api.components.v2', - syntax='proto3', - serialized_options=b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xc5\x02\n\nComponents\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12o\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\x12u\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponseB7Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2b\x06proto3' - , - dependencies=[scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.DESCRIPTOR,]) - - - - -_COMPSEARCHREQUEST = _descriptor.Descriptor( - name='CompSearchRequest', - full_name='scanoss.api.components.v2.CompSearchRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='search', full_name='scanoss.api.components.v2.CompSearchRequest.search', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='vendor', full_name='scanoss.api.components.v2.CompSearchRequest.vendor', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='component', full_name='scanoss.api.components.v2.CompSearchRequest.component', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='package', full_name='scanoss.api.components.v2.CompSearchRequest.package', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='limit', full_name='scanoss.api.components.v2.CompSearchRequest.limit', index=4, - number=6, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='offset', full_name='scanoss.api.components.v2.CompSearchRequest.offset', index=5, - number=7, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=125, - serialized_end=243, -) - - -_COMPSEARCHRESPONSE_COMPONENT = _descriptor.Descriptor( - name='Component', - full_name='scanoss.api.components.v2.CompSearchResponse.Component', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='component', full_name='scanoss.api.components.v2.CompSearchResponse.Component.component', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='purl', full_name='scanoss.api.components.v2.CompSearchResponse.Component.purl', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='url', full_name='scanoss.api.components.v2.CompSearchResponse.Component.url', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=400, - serialized_end=457, -) - -_COMPSEARCHRESPONSE = _descriptor.Descriptor( - name='CompSearchResponse', - full_name='scanoss.api.components.v2.CompSearchResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='components', full_name='scanoss.api.components.v2.CompSearchResponse.components', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='status', full_name='scanoss.api.components.v2.CompSearchResponse.status', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[_COMPSEARCHRESPONSE_COMPONENT, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=246, - serialized_end=457, -) - - -_COMPVERSIONREQUEST = _descriptor.Descriptor( - name='CompVersionRequest', - full_name='scanoss.api.components.v2.CompVersionRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='purl', full_name='scanoss.api.components.v2.CompVersionRequest.purl', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='limit', full_name='scanoss.api.components.v2.CompVersionRequest.limit', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=459, - serialized_end=508, -) - - -_COMPVERSIONRESPONSE_LICENSE = _descriptor.Descriptor( - name='License', - full_name='scanoss.api.components.v2.CompVersionResponse.License', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='scanoss.api.components.v2.CompVersionResponse.License.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='spdx_id', full_name='scanoss.api.components.v2.CompVersionResponse.License.spdx_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='is_spdx_approved', full_name='scanoss.api.components.v2.CompVersionResponse.License.is_spdx_approved', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='url', full_name='scanoss.api.components.v2.CompVersionResponse.License.url', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=666, - serialized_end=745, -) - -_COMPVERSIONRESPONSE_VERSION = _descriptor.Descriptor( - name='Version', - full_name='scanoss.api.components.v2.CompVersionResponse.Version', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='scanoss.api.components.v2.CompVersionResponse.Version.version', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='licenses', full_name='scanoss.api.components.v2.CompVersionResponse.Version.licenses', index=1, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=747, - serialized_end=847, -) - -_COMPVERSIONRESPONSE_COMPONENT = _descriptor.Descriptor( - name='Component', - full_name='scanoss.api.components.v2.CompVersionResponse.Component', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='component', full_name='scanoss.api.components.v2.CompVersionResponse.Component.component', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='purl', full_name='scanoss.api.components.v2.CompVersionResponse.Component.purl', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='url', full_name='scanoss.api.components.v2.CompVersionResponse.Component.url', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='versions', full_name='scanoss.api.components.v2.CompVersionResponse.Component.versions', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=850, - serialized_end=981, -) - -_COMPVERSIONRESPONSE = _descriptor.Descriptor( - name='CompVersionResponse', - full_name='scanoss.api.components.v2.CompVersionResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='component', full_name='scanoss.api.components.v2.CompVersionResponse.component', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='status', full_name='scanoss.api.components.v2.CompVersionResponse.status', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[_COMPVERSIONRESPONSE_LICENSE, _COMPVERSIONRESPONSE_VERSION, _COMPVERSIONRESPONSE_COMPONENT, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=511, - serialized_end=981, -) - -_COMPSEARCHRESPONSE_COMPONENT.containing_type = _COMPSEARCHRESPONSE -_COMPSEARCHRESPONSE.fields_by_name['components'].message_type = _COMPSEARCHRESPONSE_COMPONENT -_COMPSEARCHRESPONSE.fields_by_name['status'].message_type = scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._STATUSRESPONSE -_COMPVERSIONRESPONSE_LICENSE.containing_type = _COMPVERSIONRESPONSE -_COMPVERSIONRESPONSE_VERSION.fields_by_name['licenses'].message_type = _COMPVERSIONRESPONSE_LICENSE -_COMPVERSIONRESPONSE_VERSION.containing_type = _COMPVERSIONRESPONSE -_COMPVERSIONRESPONSE_COMPONENT.fields_by_name['versions'].message_type = _COMPVERSIONRESPONSE_VERSION -_COMPVERSIONRESPONSE_COMPONENT.containing_type = _COMPVERSIONRESPONSE -_COMPVERSIONRESPONSE.fields_by_name['component'].message_type = _COMPVERSIONRESPONSE_COMPONENT -_COMPVERSIONRESPONSE.fields_by_name['status'].message_type = scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._STATUSRESPONSE -DESCRIPTOR.message_types_by_name['CompSearchRequest'] = _COMPSEARCHREQUEST -DESCRIPTOR.message_types_by_name['CompSearchResponse'] = _COMPSEARCHRESPONSE -DESCRIPTOR.message_types_by_name['CompVersionRequest'] = _COMPVERSIONREQUEST -DESCRIPTOR.message_types_by_name['CompVersionResponse'] = _COMPVERSIONRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -CompSearchRequest = _reflection.GeneratedProtocolMessageType('CompSearchRequest', (_message.Message,), { - 'DESCRIPTOR' : _COMPSEARCHREQUEST, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompSearchRequest) - }) -_sym_db.RegisterMessage(CompSearchRequest) - -CompSearchResponse = _reflection.GeneratedProtocolMessageType('CompSearchResponse', (_message.Message,), { - - 'Component' : _reflection.GeneratedProtocolMessageType('Component', (_message.Message,), { - 'DESCRIPTOR' : _COMPSEARCHRESPONSE_COMPONENT, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompSearchResponse.Component) - }) - , - 'DESCRIPTOR' : _COMPSEARCHRESPONSE, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompSearchResponse) - }) -_sym_db.RegisterMessage(CompSearchResponse) -_sym_db.RegisterMessage(CompSearchResponse.Component) - -CompVersionRequest = _reflection.GeneratedProtocolMessageType('CompVersionRequest', (_message.Message,), { - 'DESCRIPTOR' : _COMPVERSIONREQUEST, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompVersionRequest) - }) -_sym_db.RegisterMessage(CompVersionRequest) - -CompVersionResponse = _reflection.GeneratedProtocolMessageType('CompVersionResponse', (_message.Message,), { - - 'License' : _reflection.GeneratedProtocolMessageType('License', (_message.Message,), { - 'DESCRIPTOR' : _COMPVERSIONRESPONSE_LICENSE, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompVersionResponse.License) - }) - , - - 'Version' : _reflection.GeneratedProtocolMessageType('Version', (_message.Message,), { - 'DESCRIPTOR' : _COMPVERSIONRESPONSE_VERSION, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompVersionResponse.Version) - }) - , - - 'Component' : _reflection.GeneratedProtocolMessageType('Component', (_message.Message,), { - 'DESCRIPTOR' : _COMPVERSIONRESPONSE_COMPONENT, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompVersionResponse.Component) - }) - , - 'DESCRIPTOR' : _COMPVERSIONRESPONSE, - '__module__' : 'scanoss.api.components.v2.scanoss_components_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.components.v2.CompVersionResponse) - }) -_sym_db.RegisterMessage(CompVersionResponse) -_sym_db.RegisterMessage(CompVersionResponse.License) -_sym_db.RegisterMessage(CompVersionResponse.Version) -_sym_db.RegisterMessage(CompVersionResponse.Component) - - -DESCRIPTOR._options = None - -_COMPONENTS = _descriptor.ServiceDescriptor( - name='Components', - full_name='scanoss.api.components.v2.Components', - file=DESCRIPTOR, - index=0, - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_start=984, - serialized_end=1309, - methods=[ - _descriptor.MethodDescriptor( - name='Echo', - full_name='scanoss.api.components.v2.Components.Echo', - index=0, - containing_service=None, - input_type=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._ECHOREQUEST, - output_type=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._ECHORESPONSE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='SearchComponents', - full_name='scanoss.api.components.v2.Components.SearchComponents', - index=1, - containing_service=None, - input_type=_COMPSEARCHREQUEST, - output_type=_COMPSEARCHRESPONSE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='GetComponentVersions', - full_name='scanoss.api.components.v2.Components.GetComponentVersions', - index=2, - containing_service=None, - input_type=_COMPVERSIONREQUEST, - output_type=_COMPVERSIONRESPONSE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), -]) -_sym_db.RegisterServiceDescriptor(_COMPONENTS) - -DESCRIPTOR.services_by_name['Components'] = _COMPONENTS - +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xc5\x02\n\nComponents\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12o\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\x12u\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponseB7Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2b\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2' + _COMPSEARCHREQUEST._serialized_start=125 + _COMPSEARCHREQUEST._serialized_end=243 + _COMPSEARCHRESPONSE._serialized_start=246 + _COMPSEARCHRESPONSE._serialized_end=457 + _COMPSEARCHRESPONSE_COMPONENT._serialized_start=400 + _COMPSEARCHRESPONSE_COMPONENT._serialized_end=457 + _COMPVERSIONREQUEST._serialized_start=459 + _COMPVERSIONREQUEST._serialized_end=508 + _COMPVERSIONRESPONSE._serialized_start=511 + _COMPVERSIONRESPONSE._serialized_end=981 + _COMPVERSIONRESPONSE_LICENSE._serialized_start=666 + _COMPVERSIONRESPONSE_LICENSE._serialized_end=745 + _COMPVERSIONRESPONSE_VERSION._serialized_start=747 + _COMPVERSIONRESPONSE_VERSION._serialized_end=847 + _COMPVERSIONRESPONSE_COMPONENT._serialized_start=850 + _COMPVERSIONRESPONSE_COMPONENT._serialized_end=981 + _COMPONENTS._serialized_start=984 + _COMPONENTS._serialized_end=1309 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index 12986d77..ac983c0e 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -2,9 +2,9 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/dependencies/v2/scanoss-dependencies.proto """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -14,447 +14,28 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor.FileDescriptor( - name='scanoss/api/dependencies/v2/scanoss-dependencies.proto', - package='scanoss.api.dependencies.v2', - syntax='proto3', - serialized_options=b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xd3\x01\n\x0c\x44\x65pendencies\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12r\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponseB;Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2b\x06proto3' - , - dependencies=[scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.DESCRIPTOR,]) - - - - -_DEPENDENCYREQUEST_PURLS = _descriptor.Descriptor( - name='Purls', - full_name='scanoss.api.dependencies.v2.DependencyRequest.Purls', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='purl', full_name='scanoss.api.dependencies.v2.DependencyRequest.Purls.purl', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='requirement', full_name='scanoss.api.dependencies.v2.DependencyRequest.Purls.requirement', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=237, - serialized_end=279, -) - -_DEPENDENCYREQUEST_FILES = _descriptor.Descriptor( - name='Files', - full_name='scanoss.api.dependencies.v2.DependencyRequest.Files', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='file', full_name='scanoss.api.dependencies.v2.DependencyRequest.Files.file', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='purls', full_name='scanoss.api.dependencies.v2.DependencyRequest.Files.purls', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=281, - serialized_end=371, -) - -_DEPENDENCYREQUEST = _descriptor.Descriptor( - name='DependencyRequest', - full_name='scanoss.api.dependencies.v2.DependencyRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='files', full_name='scanoss.api.dependencies.v2.DependencyRequest.files', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='depth', full_name='scanoss.api.dependencies.v2.DependencyRequest.depth', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[_DEPENDENCYREQUEST_PURLS, _DEPENDENCYREQUEST_FILES, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=132, - serialized_end=371, -) - - -_DEPENDENCYRESPONSE_LICENSES = _descriptor.Descriptor( - name='Licenses', - full_name='scanoss.api.dependencies.v2.DependencyResponse.Licenses', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='scanoss.api.dependencies.v2.DependencyResponse.Licenses.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='spdx_id', full_name='scanoss.api.dependencies.v2.DependencyResponse.Licenses.spdx_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='is_spdx_approved', full_name='scanoss.api.dependencies.v2.DependencyResponse.Licenses.is_spdx_approved', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='url', full_name='scanoss.api.dependencies.v2.DependencyResponse.Licenses.url', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=521, - serialized_end=601, -) - -_DEPENDENCYRESPONSE_DEPENDENCIES = _descriptor.Descriptor( - name='Dependencies', - full_name='scanoss.api.dependencies.v2.DependencyResponse.Dependencies', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='component', full_name='scanoss.api.dependencies.v2.DependencyResponse.Dependencies.component', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='purl', full_name='scanoss.api.dependencies.v2.DependencyResponse.Dependencies.purl', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='version', full_name='scanoss.api.dependencies.v2.DependencyResponse.Dependencies.version', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='licenses', full_name='scanoss.api.dependencies.v2.DependencyResponse.Dependencies.licenses', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='url', full_name='scanoss.api.dependencies.v2.DependencyResponse.Dependencies.url', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='comment', full_name='scanoss.api.dependencies.v2.DependencyResponse.Dependencies.comment', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=604, - serialized_end=774, -) - -_DEPENDENCYRESPONSE_FILES = _descriptor.Descriptor( - name='Files', - full_name='scanoss.api.dependencies.v2.DependencyResponse.Files', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='file', full_name='scanoss.api.dependencies.v2.DependencyResponse.Files.file', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='id', full_name='scanoss.api.dependencies.v2.DependencyResponse.Files.id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='status', full_name='scanoss.api.dependencies.v2.DependencyResponse.Files.status', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='dependencies', full_name='scanoss.api.dependencies.v2.DependencyResponse.Files.dependencies', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=777, - serialized_end=910, -) - -_DEPENDENCYRESPONSE = _descriptor.Descriptor( - name='DependencyResponse', - full_name='scanoss.api.dependencies.v2.DependencyResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='files', full_name='scanoss.api.dependencies.v2.DependencyResponse.files', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='status', full_name='scanoss.api.dependencies.v2.DependencyResponse.status', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[_DEPENDENCYRESPONSE_LICENSES, _DEPENDENCYRESPONSE_DEPENDENCIES, _DEPENDENCYRESPONSE_FILES, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=374, - serialized_end=910, -) - -_DEPENDENCYREQUEST_PURLS.containing_type = _DEPENDENCYREQUEST -_DEPENDENCYREQUEST_FILES.fields_by_name['purls'].message_type = _DEPENDENCYREQUEST_PURLS -_DEPENDENCYREQUEST_FILES.containing_type = _DEPENDENCYREQUEST -_DEPENDENCYREQUEST.fields_by_name['files'].message_type = _DEPENDENCYREQUEST_FILES -_DEPENDENCYRESPONSE_LICENSES.containing_type = _DEPENDENCYRESPONSE -_DEPENDENCYRESPONSE_DEPENDENCIES.fields_by_name['licenses'].message_type = _DEPENDENCYRESPONSE_LICENSES -_DEPENDENCYRESPONSE_DEPENDENCIES.containing_type = _DEPENDENCYRESPONSE -_DEPENDENCYRESPONSE_FILES.fields_by_name['dependencies'].message_type = _DEPENDENCYRESPONSE_DEPENDENCIES -_DEPENDENCYRESPONSE_FILES.containing_type = _DEPENDENCYRESPONSE -_DEPENDENCYRESPONSE.fields_by_name['files'].message_type = _DEPENDENCYRESPONSE_FILES -_DEPENDENCYRESPONSE.fields_by_name['status'].message_type = scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._STATUSRESPONSE -DESCRIPTOR.message_types_by_name['DependencyRequest'] = _DEPENDENCYREQUEST -DESCRIPTOR.message_types_by_name['DependencyResponse'] = _DEPENDENCYRESPONSE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -DependencyRequest = _reflection.GeneratedProtocolMessageType('DependencyRequest', (_message.Message,), { - - 'Purls' : _reflection.GeneratedProtocolMessageType('Purls', (_message.Message,), { - 'DESCRIPTOR' : _DEPENDENCYREQUEST_PURLS, - '__module__' : 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.dependencies.v2.DependencyRequest.Purls) - }) - , - - 'Files' : _reflection.GeneratedProtocolMessageType('Files', (_message.Message,), { - 'DESCRIPTOR' : _DEPENDENCYREQUEST_FILES, - '__module__' : 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.dependencies.v2.DependencyRequest.Files) - }) - , - 'DESCRIPTOR' : _DEPENDENCYREQUEST, - '__module__' : 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.dependencies.v2.DependencyRequest) - }) -_sym_db.RegisterMessage(DependencyRequest) -_sym_db.RegisterMessage(DependencyRequest.Purls) -_sym_db.RegisterMessage(DependencyRequest.Files) - -DependencyResponse = _reflection.GeneratedProtocolMessageType('DependencyResponse', (_message.Message,), { - - 'Licenses' : _reflection.GeneratedProtocolMessageType('Licenses', (_message.Message,), { - 'DESCRIPTOR' : _DEPENDENCYRESPONSE_LICENSES, - '__module__' : 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.dependencies.v2.DependencyResponse.Licenses) - }) - , - - 'Dependencies' : _reflection.GeneratedProtocolMessageType('Dependencies', (_message.Message,), { - 'DESCRIPTOR' : _DEPENDENCYRESPONSE_DEPENDENCIES, - '__module__' : 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.dependencies.v2.DependencyResponse.Dependencies) - }) - , - - 'Files' : _reflection.GeneratedProtocolMessageType('Files', (_message.Message,), { - 'DESCRIPTOR' : _DEPENDENCYRESPONSE_FILES, - '__module__' : 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.dependencies.v2.DependencyResponse.Files) - }) - , - 'DESCRIPTOR' : _DEPENDENCYRESPONSE, - '__module__' : 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2' - # @@protoc_insertion_point(class_scope:scanoss.api.dependencies.v2.DependencyResponse) - }) -_sym_db.RegisterMessage(DependencyResponse) -_sym_db.RegisterMessage(DependencyResponse.Licenses) -_sym_db.RegisterMessage(DependencyResponse.Dependencies) -_sym_db.RegisterMessage(DependencyResponse.Files) - - -DESCRIPTOR._options = None - -_DEPENDENCIES = _descriptor.ServiceDescriptor( - name='Dependencies', - full_name='scanoss.api.dependencies.v2.Dependencies', - file=DESCRIPTOR, - index=0, - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_start=913, - serialized_end=1124, - methods=[ - _descriptor.MethodDescriptor( - name='Echo', - full_name='scanoss.api.dependencies.v2.Dependencies.Echo', - index=0, - containing_service=None, - input_type=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._ECHOREQUEST, - output_type=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._ECHORESPONSE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), - _descriptor.MethodDescriptor( - name='GetDependencies', - full_name='scanoss.api.dependencies.v2.Dependencies.GetDependencies', - index=1, - containing_service=None, - input_type=_DEPENDENCYREQUEST, - output_type=_DEPENDENCYRESPONSE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), -]) -_sym_db.RegisterServiceDescriptor(_DEPENDENCIES) - -DESCRIPTOR.services_by_name['Dependencies'] = _DEPENDENCIES - +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xd3\x01\n\x0c\x44\x65pendencies\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12r\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponseB;Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2b\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2' + _DEPENDENCYREQUEST._serialized_start=132 + _DEPENDENCYREQUEST._serialized_end=371 + _DEPENDENCYREQUEST_PURLS._serialized_start=237 + _DEPENDENCYREQUEST_PURLS._serialized_end=279 + _DEPENDENCYREQUEST_FILES._serialized_start=281 + _DEPENDENCYREQUEST_FILES._serialized_end=371 + _DEPENDENCYRESPONSE._serialized_start=374 + _DEPENDENCYRESPONSE._serialized_end=910 + _DEPENDENCYRESPONSE_LICENSES._serialized_start=521 + _DEPENDENCYRESPONSE_LICENSES._serialized_end=601 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=604 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=774 + _DEPENDENCYRESPONSE_FILES._serialized_start=777 + _DEPENDENCYRESPONSE_FILES._serialized_end=910 + _DEPENDENCIES._serialized_start=913 + _DEPENDENCIES._serialized_end=1124 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index a70366fd..c44a5ad5 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -2,9 +2,9 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/scanning/v2/scanoss-scanning.proto """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -14,46 +14,14 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor.FileDescriptor( - name='scanoss/api/scanning/v2/scanoss-scanning.proto', - package='scanoss.api.scanning.v2', - syntax='proto3', - serialized_options=b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto2[\n\x08Scanning\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponseB3Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2b\x06proto3' - , - dependencies=[scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.DESCRIPTOR,]) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto2[\n\x08Scanning\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponseB3Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2b\x06proto3') +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: - -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - -DESCRIPTOR._options = None - -_SCANNING = _descriptor.ServiceDescriptor( - name='Scanning', - full_name='scanoss.api.scanning.v2.Scanning', - file=DESCRIPTOR, - index=0, - serialized_options=None, - create_key=_descriptor._internal_create_key, - serialized_start=119, - serialized_end=210, - methods=[ - _descriptor.MethodDescriptor( - name='Echo', - full_name='scanoss.api.scanning.v2.Scanning.Echo', - index=0, - containing_service=None, - input_type=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._ECHOREQUEST, - output_type=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2._ECHORESPONSE, - serialized_options=None, - create_key=_descriptor._internal_create_key, - ), -]) -_sym_db.RegisterServiceDescriptor(_SCANNING) - -DESCRIPTOR.services_by_name['Scanning'] = _SCANNING - + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2' + _SCANNING._serialized_start=119 + _SCANNING._serialized_end=210 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index 208e778c..53326b2b 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -2,9 +2,9 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -14,436 +14,28 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor.FileDescriptor( - name='scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto', - package='scanoss.api.vulnerabilities.v2', - syntax='proto3', - serialized_options=b'Z?github.com/scanoss/papi/api/vulnerabilitiesv2;vulnerabilitiesv2', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n Date: Tue, 7 Feb 2023 15:17:31 +0000 Subject: [PATCH 087/489] added build tags --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 4fea90ab..03b8c4eb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # SCANOSS Python Library The SCANOSS python package provides a simple, easy to consume library for interacting with SCANOSS APIs/Engine. +[![Build/Test Local Python Package](https://github.com/scanoss/scanoss.py/actions/workflows/python-local-test.yml/badge.svg)](https://github.com/scanoss/scanoss.py/actions/workflows/python-local-test.yml) +[![Build/Test Local Container](https://github.com/scanoss/scanoss.py/actions/workflows/container-local-test.yml/badge.svg)](https://github.com/scanoss/scanoss.py/actions/workflows/container-local-test.yml) + # Installation To install (from [pypi.org](https://pypi.org/project/scanoss)), please run: ```bash From b64a8f12fbfe06354c7243e23a391c6dfbdbd503 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 16:37:39 +0000 Subject: [PATCH 088/489] fixed GHCR test issue --- .github/workflows/container-publish-ghcr.yml | 2 +- README.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index e9c570ce..9e1ab564 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -83,7 +83,7 @@ jobs: run: | docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} version - docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} scan -o results.json tests + docker run -v "$(pwd)":"/scanoss" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then diff --git a/README.md b/README.md index 03b8c4eb..52f2e8a3 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # SCANOSS Python Library The SCANOSS python package provides a simple, easy to consume library for interacting with SCANOSS APIs/Engine. -[![Build/Test Local Python Package](https://github.com/scanoss/scanoss.py/actions/workflows/python-local-test.yml/badge.svg)](https://github.com/scanoss/scanoss.py/actions/workflows/python-local-test.yml) +[![Build/Test Local Package](https://github.com/scanoss/scanoss.py/actions/workflows/python-local-test.yml/badge.svg)](https://github.com/scanoss/scanoss.py/actions/workflows/python-local-test.yml) [![Build/Test Local Container](https://github.com/scanoss/scanoss.py/actions/workflows/container-local-test.yml/badge.svg)](https://github.com/scanoss/scanoss.py/actions/workflows/container-local-test.yml) +[![Publish Package - PyPI](https://github.com/scanoss/scanoss.py/actions/workflows/python-publish-pypi.yml/badge.svg)](https://github.com/scanoss/scanoss.py/actions/workflows/python-publish-pypi.yml) +[![Publish GHCR Container](https://github.com/scanoss/scanoss.py/actions/workflows/container-publish-ghcr.yml/badge.svg)](https://github.com/scanoss/scanoss.py/actions/workflows/container-publish-ghcr.yml) # Installation To install (from [pypi.org](https://pypi.org/project/scanoss)), please run: From 5e7a16853b62f1917c9f92a4604e6147e7c82167 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 17:54:05 +0000 Subject: [PATCH 089/489] install specific package version --- .github/workflows/python-publish-pypi.yml | 4 +++- .github/workflows/python-publish-testpypi.yml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index b1e53e2d..2b008e7a 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -64,7 +64,9 @@ jobs: - name: Install Remote Package run: | - pip install --upgrade scanoss + scanoss_version=$(python ./version.py) + echo "Installing scanoss ${scanoss_version}..." + pip install --upgrade scanoss==$scanoss_version which scanoss-py - name: Run Tests diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 3f8bf518..f5c16a69 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -62,7 +62,9 @@ jobs: - name: Install Remote Package run: | + scanoss_version=$(python ./version.py) pip install -r requirements.txt + echo "Install TestPyPI scanoss ${scanoss_version}..." pip install -i https://test.pypi.org/simple/ --upgrade scanoss which scanoss-py From 0105f007c61133947086f767707c715805cb8bbf Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 17:59:17 +0000 Subject: [PATCH 090/489] version display --- .github/workflows/python-publish-testpypi.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index f5c16a69..353c278f 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -29,6 +29,8 @@ jobs: - name: Install Test Package run: | pip install -r requirements.txt + scanoss_version=$(python ./version.py) + echo "Local test install of scanoss ${scanoss_version}..." pip install dist/scanoss-*-py3-none-any.whl which scanoss-py From e834d172db8f3c689b1222b47d750016071ef32a Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 7 Feb 2023 18:06:39 +0000 Subject: [PATCH 091/489] fixed dependency data in test file --- tests/data/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/requirements.txt b/tests/data/requirements.txt index db3b752f..6237a069 100644 --- a/tests/data/requirements.txt +++ b/tests/data/requirements.txt @@ -2,5 +2,5 @@ requests crc32c>=2.2 binaryornot progress -grpcio<=1.42.0 -protobuf>=3.16.0,<=3.19.1 +grpcio>1.42.0 +protobuf>3.19.1 From cf668e22d40e01de99d68ae2c3c6ac0eee12542e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 17 Feb 2023 13:55:17 +0000 Subject: [PATCH 092/489] switch to build package and setup.cfg --- .github/workflows/python-publish-pypi.yml | 22 +++++++++--- Makefile | 6 ++-- requirements-dev.txt | 1 + setup.cfg | 19 ++++++++-- setup.py | 42 ----------------------- 5 files changed, 38 insertions(+), 52 deletions(-) delete mode 100644 setup.py diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 2b008e7a..84c1b0bd 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -1,10 +1,13 @@ name: Publish Python Package - PyPI -# This workflow will upload a Python Package using Twine to PyPI when a release is created +# This workflow will upload a Python Package using Twine to PyPI and create a draft release when a tag is pushed on: workflow_dispatch: - release: - types: [ published ] + push: + tags: + - 'v*.*.*' +# release: +# types: [ published ] permissions: contents: read @@ -40,7 +43,12 @@ jobs: which scanoss-py scanoss-py version scanoss-py scan tests > results.json - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi pip uninstall -y scanoss - name: Publish Package @@ -49,6 +57,12 @@ jobs: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + - name: Create Draft Release + uses: softprops/action-gh-release@v1 + with: + draft: true + files: dist/* + test: if: success() needs: [ deploy ] diff --git a/Makefile b/Makefile index ee9340d7..37043fc0 100644 --- a/Makefile +++ b/Makefile @@ -30,17 +30,17 @@ date_time: ## Setup package build date dev_setup: date_time_clean ## Setup Python dev env for the current user @echo "Setting up dev env for the current user..." - python3 setup.py develop --user + pip3 install -e . dev_uninstall: ## Uninstall Python dev setup for the current user @echo "Uninstalling dev env..." - python3 setup.py develop --user --uninstall + pip3 uninstall -y scanoss @rm -f venv/bin/scanoss-py @rm -rf src/scanoss.egg-info dist: clean dev_uninstall date_time ## Prepare Python package into a distribution @echo "Build deployable package for distribution $(VERSION)..." - python3 setup.py sdist bdist_wheel + python3 -m build twine check dist/* publish_test: ## Publish the Python package to TestPyPI diff --git a/requirements-dev.txt b/requirements-dev.txt index 9c9c1d6c..0187cb26 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ setuptools wheel twine +build grpcio-tools diff --git a/setup.cfg b/setup.cfg index eed67f95..a6892a09 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,28 +3,41 @@ name = scanoss version = attr: scanoss.__version__ author = SCANOSS author_email = info@scanoss.com -description = Simple Python library to use the SCANOSS APIs +license = MIT +description = Simple Python library to leverage the SCANOSS APIs long_description = file: PACKAGE.md long_description_content_type = text/markdown url = https://scanoss.com project_urls = Source = https://github.com/scanoss/scanoss.py Tracker = https://github.com/scanoss/scanoss.py/issues - classifiers = Programming Language :: Python :: 3 License :: OSI Approved :: MIT Operating System :: OS Independent + Development Status :: 5 - Production/Stable [options] +packages = find_namespace: package_dir = = src -packages = find: +include_package_data = True python_requires = >=3.7 +install_requires = + requests + crc32c>=2.2 + binaryornot + progress + grpcio>1.42.0 + protobuf>3.19.1 + pypac [options.packages.find] where = src +[options.package_data] +* = data/*.txt, data/*.json + [options.entry_points] console_scripts = scanoss-py = scanoss.cli:main \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index e3070024..00000000 --- a/setup.py +++ /dev/null @@ -1,42 +0,0 @@ -import codecs -import os -from setuptools import setup - - -def read(rel_path): - here = os.path.abspath(os.path.dirname(__file__)) - with codecs.open(os.path.join(here, rel_path), 'r') as fp: - return fp.read() - - -def get_version(rel_path): - for line in read(rel_path).splitlines(): - if line.startswith('__version__'): - delim = '"' if '"' in line else "'" - return line.split(delim)[1] - else: - raise RuntimeError("Unable to find version string.") - - -setup( - name="scanoss", - version=get_version("src/scanoss/__init__.py"), - author="SCANOSS", - author_email="info@scanoss.com", - license='MIT', - description='Simple Python library to use the SCANOSS APIs.', - long_description=read("PACKAGE.md"), - long_description_content_type='text/markdown', - install_requires=["requests", # TODO Add min req for python 3.10 here - urllib3>=1.26.8 and requests>=2.27.0? - "crc32c>=2.2", "binaryornot", "progress", "grpcio>1.42.0", - "protobuf>3.19.1", "pypac" - ], - include_package_data=True, - package_data={'': ['data/*.json', 'data/*.txt']}, - classifiers=[ - "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3", - "Operating System :: OS Independent" - ], - python_requires='>=3.7' -) From f529034c0fa01c4bd919b005cde02b040ed4a428 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 28 Feb 2023 18:49:38 +0000 Subject: [PATCH 093/489] added fast winnowing module integration --- .github/workflows/container-local-test.yml | 1 + .github/workflows/container-publish-ghcr.yml | 3 +-- .github/workflows/python-local-test.yml | 16 ++++++++++++- .github/workflows/python-publish-pypi.yml | 21 ++++++++++++---- .github/workflows/python-publish-testpypi.yml | 24 +++++++++++++++++-- Dockerfile | 2 ++ PACKAGE.md | 10 ++++++++ setup.cfg | 7 +++++- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 23 +++++++++++++++--- src/scanoss/scanner.py | 15 +++++++++++- 11 files changed, 109 insertions(+), 15 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 525c7801..cf112a21 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -53,6 +53,7 @@ jobs: docker load --input /tmp/scanoss-py.tar docker image ls -a docker run ${{ env.IMAGE_NAME }} version + docker run ${{ env.IMAGE_NAME }} fast docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 9e1ab564..5e58dedc 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -6,8 +6,6 @@ on: push: tags: - 'v*.*.*' -# release: -# types: [published] env: REGISTRY: ghcr.io @@ -83,6 +81,7 @@ jobs: run: | docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} version + docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} fast docker run -v "$(pwd)":"/scanoss" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 98c4f71a..5d5bc793 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -16,7 +16,6 @@ permissions: jobs: build: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 @@ -43,6 +42,21 @@ jobs: run: | which scanoss-py scanoss-py version + scanoss-py fast + scanoss-py scan tests > results.json + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi + + - name: Run Tests (fast winnowing) + run: | + pip install scanoss_winnowing + which scanoss-py + scanoss-py version + scanoss-py fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 84c1b0bd..7fe38a80 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -6,8 +6,6 @@ on: push: tags: - 'v*.*.*' -# release: -# types: [ published ] permissions: contents: read @@ -15,7 +13,6 @@ permissions: jobs: deploy: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 @@ -42,6 +39,7 @@ jobs: run: | which scanoss-py scanoss-py version + scanoss-py fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" @@ -54,6 +52,7 @@ jobs: - name: Publish Package uses: pypa/gh-action-pypi-publish@release/v1 with: +# skip_existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} @@ -67,7 +66,6 @@ jobs: if: success() needs: [ deploy ] runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 @@ -87,6 +85,21 @@ jobs: run: | which scanoss-py scanoss-py version + scanoss-py fast + scanoss-py scan tests > results.json + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi + + - name: Run Tests (fast winnowing) + run: | + pip install scanoss_winnowing + which scanoss-py + scanoss-py version + scanoss-py fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 353c278f..49599f3d 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -9,7 +9,6 @@ permissions: jobs: deploy: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 @@ -39,12 +38,18 @@ jobs: which scanoss-py scanoss-py version scanoss-py scan tests > results.json - echo "ID Count: $(cat results.json | grep '"id":' | wc -l)" + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi pip uninstall -y scanoss - name: Publish Test Package uses: pypa/gh-action-pypi-publish@release/v1 with: + skip_existing: true user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository_url: https://test.pypi.org/legacy/ @@ -74,6 +79,21 @@ jobs: run: | which scanoss-py scanoss-py version + scanoss-py fast + scanoss-py scan tests > results.json + id_count=$(cat results.json | grep '"id":' | wc -l) + echo "ID Count: $id_count" + if [[ $id_count -lt 1 ]]; then + echo "Error: Scan test did not produce any results. Failing" + exit 1 + fi + + - name: Run Tests (fast winnowing) + run: | + pip install scanoss_winnowing + which scanoss-py + scanoss-py version + scanoss-py fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/Dockerfile b/Dockerfile index 2e28cce9..f521aa60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,9 @@ ENV PATH=/root/.local/bin:$PATH COPY ./dist/scanoss-*-py3-none-any.whl /install/ +# Install dependencies RUN pip3 install --user /install/scanoss-*-py3-none-any.whl +RUN pip3 install --user scanoss_winnowing RUN pip3 install --user scancode-toolkit-mini #RUN pip3 install --user typecode-libmagic diff --git a/PACKAGE.md b/PACKAGE.md index 3987ae42..84d46f40 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -11,6 +11,16 @@ To upgrade an existing installation please run: pip3 install --upgrade scanoss ``` +### Fast Winnowing +To take advantage of faster fingerprinting, please install the optional [scanoss_winnowing](https://pypi.org/project/scanoss_winnowing/) package: +```bash +pip3 install scanoss_winnowing +``` +Or directly using: +```bash +pip3 install scanoss[fast_winnowing] +``` + ### Docker Alternatively, there is a docker image of the compiled package. It can be found [here](https://github.com/scanoss/scanoss.py/pkgs/container/scanoss-py). Details of how to run it can be found [here](https://github.com/scanoss/scanoss.py/blob/main/GHCR.md). diff --git a/setup.cfg b/setup.cfg index a6892a09..070d26bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,9 +13,10 @@ project_urls = Tracker = https://github.com/scanoss/scanoss.py/issues classifiers = Programming Language :: Python :: 3 - License :: OSI Approved :: MIT + License :: OSI Approved :: MIT License Operating System :: OS Independent Development Status :: 5 - Production/Stable + Programming Language :: Python :: 3 [options] packages = find_namespace: @@ -32,6 +33,10 @@ install_requires = protobuf>3.19.1 pypac +[options.extras_require] +fast_winnowing = + scanoss_winnowing + [options.packages.find] where = src diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b284dcf7..39a8ba5a 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.7' +__version__ = '1.3.8' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 0c0d1b1e..45e24f44 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -36,6 +36,7 @@ from .spdxlite import SpdxLite from .csvoutput import CsvOutput from . import __version__ +from .scanner import FAST_WINNOWING def print_stderr(*args, **kwargs): @@ -49,7 +50,7 @@ def setup_args() -> None: """ Setup all the command line arguments for processing """ - parser = argparse.ArgumentParser(description=f'SCANOSS Python CLI. Ver: {__version__}, License: MIT') + parser = argparse.ArgumentParser(description=f'SCANOSS Python CLI. Ver: {__version__}, License: MIT, Fast Winnowing: {FAST_WINNOWING}') parser.add_argument('--version', '-v', action='store_true', help='Display version details') subparsers = parser.add_subparsers(title='Sub Commands', dest='subparser', description='valid subcommands', @@ -59,6 +60,12 @@ def setup_args() -> None: p_ver = subparsers.add_parser('version', aliases=['ver'], description=f'Version of SCANOSS CLI: {__version__}', help='SCANOSS version') p_ver.set_defaults(func=ver) + + # Sub-command: fast + p_ver = subparsers.add_parser('fast', + description=f'Is fast winnowing enabled: {__version__}', help='SCANOSS fast winnowing') + p_ver.set_defaults(func=fast) + # Sub-command: scan p_scan = subparsers.add_parser('scan', aliases=['sc'], description=f'Analyse/scan the given source base: {__version__}', @@ -118,6 +125,9 @@ def setup_args() -> None: p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') + p_wfp.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions') + p_wfp.add_argument('--all-folders', action='store_true', help='Fingerprint all folders') + p_wfp.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders') # Sub-command: dependency p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], @@ -232,7 +242,6 @@ def setup_args() -> None: exit(1) args.func(parser, args) # Execute the function associated with the sub-command - def ver(*_): """ Run the "ver" sub-command @@ -240,6 +249,12 @@ def ver(*_): """ print(f'Version: {__version__}') +def fast(*_): + """ + Run the "fast" sub-command + :param _: ignored/unused + """ + print(f'Fast Winnowing: {FAST_WINNOWING}') def file_count(parser, args): """ @@ -293,7 +308,9 @@ def wfp(parser, args): open(scan_output, 'w').close() scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value # Skip snippet generation or not - scanner = Scanner(debug=args.debug, quiet=args.quiet, obfuscate=args.obfuscate, scan_options=scan_options) + scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, obfuscate=args.obfuscate, + scan_options=scan_options, all_extensions=args.all_extensions, + all_folders=args.all_folders, hidden_files_folders=args.all_hidden) if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index f8a87da1..c3983371 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -32,7 +32,6 @@ from pypac.parser import PACFile from .scanossapi import ScanossApi -from .winnowing import Winnowing from .cyclonedx import CycloneDx from .spdxlite import SpdxLite from .csvoutput import CsvOutput @@ -42,6 +41,13 @@ from .scanossgrpc import ScanossGrpc from .scantype import ScanType from .scanossbase import ScanossBase +FAST_WINNOWING = False +try: + from scanoss_winnowing.winnowing import Winnowing + FAST_WINNOWING = True +except ModuleNotFoundError or ImportError: + FAST_WINNOWING = False + from .winnowing import Winnowing from . import __version__ @@ -799,6 +805,9 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): wfps = '' scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir)+1 self.print_msg(f'Searching {scan_dir} for files to fingerprint...') + spinner = None + if not self.quiet and self.isatty: + spinner = Spinner('Fingerprinting ') for root, dirs, files in os.walk(scan_dir): dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories filtered_files = self.__filter_files(files) # Strip out unwanted files @@ -812,7 +821,11 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink if f_size > 0: # Ignore empty files self.print_debug(f'Fingerprinting {path}...') + if spinner: + spinner.next() wfps += self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) + if spinner: + spinner.finish() if wfps: if wfp_file: self.print_stderr(f'Writing fingerprints to {wfp_file}') From 58c580873f047cec433acd06bda7c245585f57d6 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 28 Feb 2023 19:01:50 +0000 Subject: [PATCH 094/489] added fast winnowing module integration --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 39a8ba5a..19a16565 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.8' +__version__ = '1.3.9' From 08596491d87620074c201171fc10c039298cc9ae Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 28 Feb 2023 19:06:49 +0000 Subject: [PATCH 095/489] fix testpypi version install --- .github/workflows/python-publish-testpypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 49599f3d..f6eb6419 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -72,7 +72,7 @@ jobs: scanoss_version=$(python ./version.py) pip install -r requirements.txt echo "Install TestPyPI scanoss ${scanoss_version}..." - pip install -i https://test.pypi.org/simple/ --upgrade scanoss + pip install -i https://test.pypi.org/simple/ --upgrade scanoss==${scanoss_version} which scanoss-py - name: Run Tests From 860d155f21b1c57d547e1c6daa3c4015bb634fff Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 1 Mar 2023 14:32:01 +0000 Subject: [PATCH 096/489] added official support for fast winnowing --- CHANGELOG.md | 8 ++++++++ CLIENT_HELP.md | 11 +++++++++++ src/scanoss/__init__.py | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc113c0e..c44c9dc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.4.0] - 2023-01-01 +### Added +- Added support for fast winnowing (15x improvement) thanks to a contribution from [tardyp](https://github.com/tardyp) + - This is enabled by a supporting package; [scanoss_winnowing](https://github.com/scanoss/scanoss-winnowing.py). + - It can be installed using: `pip3 install scanoss_winnowing` + - Or using: `pip3 install --upgrade scanoss[fast_winnowing]` + ## [1.3.7] - 2023-02-07 ### Added - Upgrade to the latest protobuf and grpcio packages @@ -201,3 +208,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.5]: https://github.com/scanoss/scanoss.py/compare/v1.3.4...v1.3.5 [1.3.6]: https://github.com/scanoss/scanoss.py/compare/v1.3.5...v1.3.6 [1.3.7]: https://github.com/scanoss/scanoss.py/compare/v1.3.6...v1.3.7 +[1.4.0]: https://github.com/scanoss/scanoss.py/compare/v1.3.7...v1.4.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 5b04d373..58390091 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -102,3 +102,14 @@ Simply run the following commands find out: * `scanoss-py utils pac-proxy --pac file://proxy.pac --url https://osskb.org` * url * `scanoss-py utils pac-proxy --pac https://path.to/proxy.pac --url https://osskb.org` + +## GRPCIO Library installation for Apple Silicon (before 1.5.3) +Versions of [grpcio](https://pypi.org/project/grpcio) prior to `1.5.3` did not contain a binary wheel for Apple Silicon. + +[Pietro De Nicolao](https://github.com/pietrodn) has kindly created a [GitHub repo](https://github.com/pietrodn/grpcio-mac-arm-build) to build the M1/M2 compatible wheels. +Simply browse to the [releases](https://github.com/pietrodn/grpcio-mac-arm-build/releases) area, choose the desired release version and install the wheel matching your python version: +```bash +pip3 install --upgrade https://github.com/pietrodn/grpcio-mac-arm-build/releases/download/1.51.1/grpcio-1.51.1-cp39-cp39-macosx_11_0_arm64.whl +``` + +This command above will install `grpcio` `1.5.1` for Python `3.9`. To install for `3.10` simply replace the `cp39` with `cp310`. diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 19a16565..6f752133 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.3.9' +__version__ = '1.4.0' From a292d4366fea1b11b5092eb821d9155d266a204c Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 1 Mar 2023 14:44:15 +0000 Subject: [PATCH 097/489] fix workflow permissions --- .github/workflows/python-publish-pypi.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 7fe38a80..8d8e4744 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -7,9 +7,6 @@ on: tags: - 'v*.*.*' -permissions: - contents: read - jobs: deploy: runs-on: ubuntu-latest @@ -26,7 +23,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt - - name: Build Package + - name: Build Package - ${{ github.ref_name }} run: make dist - name: Install Test Package @@ -49,14 +46,15 @@ jobs: fi pip uninstall -y scanoss - - name: Publish Package + - name: Publish Package - ${{ github.ref_name }} uses: pypa/gh-action-pypi-publish@release/v1 with: -# skip_existing: true + skip_existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - - name: Create Draft Release + - name: Create Draft Release ${{ github.ref_type }} - ${{ github.ref_name }} + if: github.ref_type == 'tag' && startsWith(github.ref_name, 'v') uses: softprops/action-gh-release@v1 with: draft: true From edbc41935fabe514ad5fd256345bdb6f6f84ec4e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Wed, 1 Mar 2023 14:55:28 +0000 Subject: [PATCH 098/489] disable skip existing for package upload --- .github/workflows/python-publish-pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 8d8e4744..60d3bf40 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -49,7 +49,7 @@ jobs: - name: Publish Package - ${{ github.ref_name }} uses: pypa/gh-action-pypi-publish@release/v1 with: - skip_existing: true +# skip_existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} From 1122e2e6ccd7b0ea88b910346a4311a364e90894 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 9 Mar 2023 17:54:32 +0000 Subject: [PATCH 099/489] fixed custom certificate issue for scanning --- src/scanoss/__init__.py | 2 +- src/scanoss/scanossapi.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 6f752133..43299f66 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.4.0' +__version__ = '1.4.1' diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 4c9e4b62..5d72c3dd 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -109,8 +109,7 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st self.session.verify = False elif ca_cert: self.verify = ca_cert - self.session.cert = ca_cert - self.session.verify = True + self.session.verify = ca_cert self.proxies = {'https': proxy, 'http': proxy} if proxy else None if self. proxies: self.session.proxies = self.proxies From bbf40ea12278cb8c681cc17a7eb2cccdce6ad6f5 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 9 Mar 2023 18:09:34 +0000 Subject: [PATCH 100/489] added support for certificate chain download --- CHANGELOG.md | 11 ++++++++- cert_download.sh | 2 +- requirements.txt | 3 ++- setup.cfg | 1 + src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 53 +++++++++++++++++++---------------------- 6 files changed, 40 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c44c9dc1..127ef14f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.4.0] - 2023-01-01 +## [1.4.2] - 2023-03-09 +### Fixed +- Fixed issue with custom certificate when scanning (--ca-cert) +### Added +- Added support to download full certificate chain with: + - `cert_download.sh` + - `scanoss-py utils cdl` + +## [1.4.0] - 2023-03-01 ### Added - Added support for fast winnowing (15x improvement) thanks to a contribution from [tardyp](https://github.com/tardyp) - This is enabled by a supporting package; [scanoss_winnowing](https://github.com/scanoss/scanoss-winnowing.py). @@ -209,3 +217,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.6]: https://github.com/scanoss/scanoss.py/compare/v1.3.5...v1.3.6 [1.3.7]: https://github.com/scanoss/scanoss.py/compare/v1.3.6...v1.3.7 [1.4.0]: https://github.com/scanoss/scanoss.py/compare/v1.3.7...v1.4.0 +[1.4.2]: https://github.com/scanoss/scanoss.py/compare/v1.4.0...v1.4.2 diff --git a/cert_download.sh b/cert_download.sh index f2fd528e..d9ef9f7d 100755 --- a/cert_download.sh +++ b/cert_download.sh @@ -100,4 +100,4 @@ if [ $force -eq 0 ] && [ -f "$pemfile" ] ; then fi echo "Attempting to get PEM certificate from $host:$port and saving to $pemfile ..." -openssl s_client -showcerts -connect "$host:$port" -servername "$host" /dev/null | openssl x509 -outform PEM > "$pemfile" +openssl s_client -showcerts -verify 5 -connect "$host:$port" -servername "$host" < /dev/null 2> /dev/null | awk '/BEGIN/,/END/{ if(/BEGIN/){a++}; print}' > "$pemfile" diff --git a/requirements.txt b/requirements.txt index f6ad3cf2..33783dd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ progress grpcio>1.42.0 protobuf>3.19.1 pypac -urllib3 \ No newline at end of file +urllib3 +pyOpenSSL \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 070d26bb..92e4f0fb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,7 @@ install_requires = grpcio>1.42.0 protobuf>3.19.1 pypac + pyOpenSSL [options.extras_require] fast_winnowing = diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 43299f66..ac053c5f 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.4.1' +__version__ = '1.4.2' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 45e24f44..a3942dfc 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -561,43 +561,40 @@ def utils_cert_download(_, args): :param _: ignore/unused :param args: Parsed arguments """ - import ssl from urllib.parse import urlparse import socket + from OpenSSL import SSL, crypto import traceback file = sys.stdout hostname = 'unset' port = 'unkown' + if args.output: + file = open(args.output, 'w') + parsed_url = urlparse(args.hostname) + hostname = parsed_url.hostname or args.hostname # Use the parse hostname, or it None use the supplied one + port = int(parsed_url.port or args.port) # Use the parsed port, if not use the supplied one (default 443) + certs = [] try: - if args.output: - file = open(args.output, 'w') - parsed_url = urlparse(args.hostname) - hostname = parsed_url.hostname or args.hostname # Use the parse hostname, or it None use the supplied one - port = int(parsed_url.port or args.port) # Use the parsed port, if not use the supplied one (default 443) - conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - sock = context.wrap_socket(conn, server_hostname=hostname) - if not args.quiet or args.debug: - print_stderr(f'Attempting to download PEM certificate from {hostname}:{port} ...') if args.debug: - print_stderr('Connecting to host...') - sock.connect((hostname, port)) - if args.debug: - print_stderr('Getting peer cert...') - peer_cert = sock.getpeercert(True) - if not peer_cert: - print_stderr(f'Error: Failed to download peer certificate data from {hostname}:{port}') - exit(1) - if args.debug: - print_stderr('Converting DER to PEM...') - cert_data = ssl.DER_cert_to_PEM_cert(peer_cert) - if not cert_data or cert_data == '': - print_stderr(f'Error: Failed to convert certificate data to PEM from {hostname}:{port}') - exit(1) - else: - print(cert_data.strip(), file=file) # Print the downloaded PEM certificate - except Exception as e: + print_stderr(f'Connecting to {hostname} on {port}...') + conn = SSL.Connection(SSL.Context(SSL.TLSv1_2_METHOD), socket.socket()) + conn.connect((hostname, port)) + conn.do_handshake() + certs = conn.get_peer_cert_chain() + for index, cert in enumerate(certs): + cert_components = dict(cert.get_subject().get_components()) + if(sys.version_info[0] >= 3): + cn = cert_components.get(b'CN') + else: + cn = cert_components.get('CN') + if not args.quiet: + print_stderr(f'Centificate {index} - CN: {cn}') + if(sys.version_info[0] >= 3): + print((crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file) # Print the downloaded PEM certificate + else: + print((crypto.dump_certificate(crypto.FILETYPE_PEM, cert)).strip(), file=file) + except SSL.Error as e: print_stderr(f'ERROR: Exception ({e.__class__.__name__}) Downloading certificate from {hostname}:{port} - {e}.') if args.debug: traceback.print_exc() From 6d49bba9f27afc242d047fa9eb98a792b86ce1e2 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 10 Mar 2023 09:23:32 +0000 Subject: [PATCH 101/489] updates sbom --- .github/workflows/python-publish-pypi.yml | 2 + SBOM.json | 204 +++++++++++++++++----- 2 files changed, 161 insertions(+), 45 deletions(-) diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 60d3bf40..a23cf80e 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -75,6 +75,8 @@ jobs: - name: Install Remote Package run: | scanoss_version=$(python ./version.py) + echo "Sleeping before checking PyPI for new release version ${scanoss_version}..." + sleep 60 echo "Installing scanoss ${scanoss_version}..." pip install --upgrade scanoss==$scanoss_version which scanoss-py diff --git a/SBOM.json b/SBOM.json index 7f6d85d0..c21c88d9 100644 --- a/SBOM.json +++ b/SBOM.json @@ -1,45 +1,159 @@ -[ - { - "vendor": "scanoss", - "component": "scanoss.py", - "purl": "pkg:github/scanoss/scanoss.py", - "dependency": "self", - "comment": "scanoss.py is the implementation, everything else is dependencies", - "license": "MIT", - "license_url": "https://raw.githubusercontent.com/scanoss/scanoss.py/main/LICENSE" - }, - { - "vendor": "ICRAR", - "component": "crc32c", - "dependency": "runtime", - "homepage": "https://github.com/ICRAR/crc32c", - "purl": "pkg:github/ICRAR/crc32c", - "license_url": "https://raw.githubusercontent.com/ICRAR/crc32c/master/LICENSE", - "license": "LGPL-2.1-only" - }, - { - "vendor": "pst", - "component": "requests", - "dependency": "runtime", - "homepage": "https://github.com/psf/requests", - "purl": "pkg:github/psf/requests", - "license_url": "https://raw.githubusercontent.com/psf/requests/master/LICENSE", - "license": "Apache-2.0" - }, - { - "vendor": "audreyfeldroy", - "component": "binaryornot", - "dependency": "runtime", - "homepage": "https://github.com/audreyfeldroy/binaryornot", - "license": "BSD-3-Clause" - }, - { - "vendor": "verigak", - "component": "progress", - "dependency": "runtime", - "homepage": "https://github.com/verigak/progress", - "purl": "pkg:github/verigak/progress", - "license_url": "https://opensource.org/licenses/ISC", - "license": "ISC" - } -] +{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "serialNumber": "urn:uuid:12fa7d1c-20bd-4b2c-8ed3-b2609cb6f7fb", + "version": 1, + "components": [ + { + "type": "library", + "name": "requests", + "publisher": "psf", + "version": "2.28.2", + "purl": "pkg:pypi/requests", + "bom-ref": "pkg:pypi/requests", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ] + }, + { + "type": "library", + "name": "crc32c", + "publisher": "ICRAR", + "version": "2.3", + "purl": "pkg:pypi/crc32c", + "bom-ref": "pkg:pypi/crc32c", + "licenses": [ + { + "license": { + "name": "LGPL-2.1-only" + } + } + ] + }, + { + "type": "library", + "name": "binaryornot", + "publisher": "audreyfeldroy", + "version": "0.4.4", + "purl": "pkg:pypi/binaryornot", + "bom-ref": "pkg:pypi/binaryornot", + "licenses": [ + { + "license": { + "id": "BSD-1-Clause" + } + } + ] + }, + { + "type": "library", + "name": "progress", + "publisher": "verigak", + "version": "1.6", + "purl": "pkg:pypi/progress", + "bom-ref": "pkg:pypi/progress", + "licenses": [ + { + "license": { + "id": "ISC" + } + } + ] + }, + { + "type": "library", + "name": "grpcio", + "publisher": "google", + "version": "1.51.3", + "purl": "pkg:pypi/grpcio", + "bom-ref": "pkg:pypi/grpcio", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ] + }, + { + "type": "library", + "name": "protobuf", + "publisher": "google", + "version": "4.22.1", + "purl": "pkg:pypi/protobuf", + "bom-ref": "pkg:pypi/protobuf", + "licenses": [ + { + "license": { + "name": "BSD-3-Clause" + } + } + ] + }, + { + "type": "library", + "name": "PyPAC", + "publisher": "carsonyl", + "version": "0.16.1", + "purl": "pkg:pypi/pypac", + "bom-ref": "pkg:pypi/pypac", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ] + }, + { + "type": "library", + "name": "pyopenssl", + "publisher": "pyca", + "version": "23.0.0", + "purl": "pkg:pypi/pyopenssl", + "bom-ref": "pkg:pypi/pyopenssl", + "licenses": [ + { + "license": { + "id": "Apache-2.0" + } + } + ] + }, + { + "type": "library", + "name": "scanoss-winnowing", + "publisher": "scanoss", + "version": "0.1.0", + "purl": "pkg:pypi/scanoss-winnowing", + "bom-ref": "pkg:pypi/scanoss-winnowing", + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ] + }, + { + "type": "library", + "name": "scanoss.py", + "publisher": "scanoss", + "version": "0.9.0", + "purl": "pkg:github/scanoss/scanoss.py", + "bom-ref": "pkg:github/scanoss/scanoss.py", + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ] + } + ], + "vulnerabilities": [] +} From 0b49c69e581509c4fae5f1732a5d89904f0d21e2 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 10 Mar 2023 10:58:57 +0000 Subject: [PATCH 102/489] added latest PAPI api interfaces --- .../api/common/v2/scanoss_common_pb2.py | 10 +- .../components/v2/scanoss_components_pb2.py | 48 ++++---- .../v2/scanoss_cryptography_pb2.py | 39 +++++++ .../v2/scanoss_cryptography_pb2_grpc.py | 108 ++++++++++++++++++ .../v2/scanoss_dependencies_pb2.py | 42 ++++--- .../api/scanning/v2/scanoss_scanning_pb2.py | 12 +- .../v2/scanoss_vulnerabilities_pb2.py | 44 ++++--- 7 files changed, 240 insertions(+), 63 deletions(-) create mode 100644 src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py create mode 100644 src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index 56fee94b..23546c71 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) @@ -21,12 +21,16 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _STATUSCODE._serialized_start=220 - _STATUSCODE._serialized_end=316 + _STATUSCODE._serialized_start=336 + _STATUSCODE._serialized_end=432 _STATUSRESPONSE._serialized_start=69 _STATUSRESPONSE._serialized_end=153 _ECHOREQUEST._serialized_start=155 _ECHOREQUEST._serialized_end=185 _ECHORESPONSE._serialized_start=187 _ECHORESPONSE._serialized_end=218 + _PURLREQUEST._serialized_start=220 + _PURLREQUEST._serialized_end=334 + _PURLREQUEST_PURLS._serialized_start=292 + _PURLREQUEST_PURLS._serialized_end=334 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index 8ae503da..2ef8988a 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -12,32 +12,40 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xc5\x02\n\nComponents\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12o\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\x12u\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponseB7Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xb9\x03\n\nComponents\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/components/echo:\x01*\x12\x95\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/components/search:\x01*\x12\x9d\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"&\x82\xd3\xe4\x93\x02 \"\x1b/api/v2/components/versions:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Components Service\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2' - _COMPSEARCHREQUEST._serialized_start=125 - _COMPSEARCHREQUEST._serialized_end=243 - _COMPSEARCHRESPONSE._serialized_start=246 - _COMPSEARCHRESPONSE._serialized_end=457 - _COMPSEARCHRESPONSE_COMPONENT._serialized_start=400 - _COMPSEARCHRESPONSE_COMPONENT._serialized_end=457 - _COMPVERSIONREQUEST._serialized_start=459 - _COMPVERSIONREQUEST._serialized_end=508 - _COMPVERSIONRESPONSE._serialized_start=511 - _COMPVERSIONRESPONSE._serialized_end=981 - _COMPVERSIONRESPONSE_LICENSE._serialized_start=666 - _COMPVERSIONRESPONSE_LICENSE._serialized_end=745 - _COMPVERSIONRESPONSE_VERSION._serialized_start=747 - _COMPVERSIONRESPONSE_VERSION._serialized_end=847 - _COMPVERSIONRESPONSE_COMPONENT._serialized_start=850 - _COMPVERSIONRESPONSE_COMPONENT._serialized_end=981 - _COMPONENTS._serialized_start=984 - _COMPONENTS._serialized_end=1309 + DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\331\001\022s\n\032SCANOSS Components Service\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _COMPONENTS.methods_by_name['Echo']._options = None + _COMPONENTS.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/components/echo:\001*' + _COMPONENTS.methods_by_name['SearchComponents']._options = None + _COMPONENTS.methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/components/search:\001*' + _COMPONENTS.methods_by_name['GetComponentVersions']._options = None + _COMPONENTS.methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002 \"\033/api/v2/components/versions:\001*' + _COMPSEARCHREQUEST._serialized_start=201 + _COMPSEARCHREQUEST._serialized_end=319 + _COMPSEARCHRESPONSE._serialized_start=322 + _COMPSEARCHRESPONSE._serialized_end=533 + _COMPSEARCHRESPONSE_COMPONENT._serialized_start=476 + _COMPSEARCHRESPONSE_COMPONENT._serialized_end=533 + _COMPVERSIONREQUEST._serialized_start=535 + _COMPVERSIONREQUEST._serialized_end=584 + _COMPVERSIONRESPONSE._serialized_start=587 + _COMPVERSIONRESPONSE._serialized_end=1057 + _COMPVERSIONRESPONSE_LICENSE._serialized_start=742 + _COMPVERSIONRESPONSE_LICENSE._serialized_end=821 + _COMPVERSIONRESPONSE_VERSION._serialized_start=823 + _COMPVERSIONRESPONSE_VERSION._serialized_end=923 + _COMPVERSIONRESPONSE_COMPONENT._serialized_start=926 + _COMPVERSIONRESPONSE_COMPONENT._serialized_end=1057 + _COMPONENTS._serialized_start=1060 + _COMPONENTS._serialized_end=1501 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py new file mode 100644 index 00000000..3431e10e --- /dev/null +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: scanoss/api/cryptography/v2/scanoss-cryptography.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xb9\x02\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x31\n\nAlgorithms\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\x1au\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12M\n\nalgorithms\x18\x03 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.AlgorithmResponse.Algorithms2\x97\x02\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/cryptography/algorithms:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _CRYPTOGRAPHY.methods_by_name['Echo']._options = None + _CRYPTOGRAPHY.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/cryptography/echo:\001*' + _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._options = None + _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/cryptography/algorithms:\001*' + _ALGORITHMRESPONSE._serialized_start=208 + _ALGORITHMRESPONSE._serialized_end=521 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_start=353 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_end=402 + _ALGORITHMRESPONSE_PURLS._serialized_start=404 + _ALGORITHMRESPONSE_PURLS._serialized_end=521 + _CRYPTOGRAPHY._serialized_start=524 + _CRYPTOGRAPHY._serialized_end=803 +# @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py new file mode 100644 index 00000000..1d641e82 --- /dev/null +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py @@ -0,0 +1,108 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from scanoss.api.cryptography.v2 import scanoss_cryptography_pb2 as scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2 + + +class CryptographyStub(object): + """ + Expose all of the SCANOSS Cryptography RPCs here + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Echo = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) + self.GetAlgorithms = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, + ) + + +class CryptographyServicer(object): + """ + Expose all of the SCANOSS Cryptography RPCs here + """ + + def Echo(self, request, context): + """Standard echo + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetAlgorithms(self, request, context): + """Get Cryptographic algorithms associated with a list of PURLs + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_CryptographyServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetAlgorithms': grpc.unary_unary_rpc_method_handler( + servicer.GetAlgorithms, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Cryptography(object): + """ + Expose all of the SCANOSS Cryptography RPCs here + """ + + @staticmethod + def Echo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/Echo', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetAlgorithms(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index ac983c0e..0090b4cd 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -12,30 +12,36 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xd3\x01\n\x0c\x44\x65pendencies\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12r\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponseB;Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xa8\x02\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2' - _DEPENDENCYREQUEST._serialized_start=132 - _DEPENDENCYREQUEST._serialized_end=371 - _DEPENDENCYREQUEST_PURLS._serialized_start=237 - _DEPENDENCYREQUEST_PURLS._serialized_end=279 - _DEPENDENCYREQUEST_FILES._serialized_start=281 - _DEPENDENCYREQUEST_FILES._serialized_end=371 - _DEPENDENCYRESPONSE._serialized_start=374 - _DEPENDENCYRESPONSE._serialized_end=910 - _DEPENDENCYRESPONSE_LICENSES._serialized_start=521 - _DEPENDENCYRESPONSE_LICENSES._serialized_end=601 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=604 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=774 - _DEPENDENCYRESPONSE_FILES._serialized_start=777 - _DEPENDENCYRESPONSE_FILES._serialized_end=910 - _DEPENDENCIES._serialized_start=913 - _DEPENDENCIES._serialized_end=1124 + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service\"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _DEPENDENCIES.methods_by_name['Echo']._options = None + _DEPENDENCIES.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' + _DEPENDENCIES.methods_by_name['GetDependencies']._options = None + _DEPENDENCIES.methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' + _DEPENDENCYREQUEST._serialized_start=208 + _DEPENDENCYREQUEST._serialized_end=447 + _DEPENDENCYREQUEST_PURLS._serialized_start=313 + _DEPENDENCYREQUEST_PURLS._serialized_end=355 + _DEPENDENCYREQUEST_FILES._serialized_start=357 + _DEPENDENCYREQUEST_FILES._serialized_end=447 + _DEPENDENCYRESPONSE._serialized_start=450 + _DEPENDENCYRESPONSE._serialized_end=986 + _DEPENDENCYRESPONSE_LICENSES._serialized_start=597 + _DEPENDENCYRESPONSE_LICENSES._serialized_end=677 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=680 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=850 + _DEPENDENCYRESPONSE_FILES._serialized_start=853 + _DEPENDENCYRESPONSE_FILES._serialized_end=986 + _DEPENDENCIES._serialized_start=989 + _DEPENDENCIES._serialized_end=1285 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index c44a5ad5..60c795dd 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -12,16 +12,20 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto2[\n\x08Scanning\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponseB3Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto2}\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2' - _SCANNING._serialized_start=119 - _SCANNING._serialized_end=210 + DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service\"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _SCANNING.methods_by_name['Echo']._options = None + _SCANNING.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' + _SCANNING._serialized_start=195 + _SCANNING._serialized_end=320 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index 53326b2b..9fc87ed3 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -12,30 +12,38 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n Date: Fri, 10 Mar 2023 11:33:17 +0000 Subject: [PATCH 103/489] added crypto PAPI api interface --- .../api/common/v2/scanoss_common_pb2.py | 10 +- .../components/v2/scanoss_components_pb2.py | 4 +- .../v2/scanoss_components_pb2_grpc.py | 12 ++ .../v2/scanoss_cryptography_pb2.py | 33 +++ .../v2/scanoss_cryptography_pb2_grpc.py | 194 ++++++++++++++++++ .../v2/scanoss_dependencies_pb2.py | 4 +- .../v2/scanoss_dependencies_pb2_grpc.py | 92 ++++++++- .../api/scanning/v2/scanoss_scanning_pb2.py | 4 +- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 4 + .../v2/scanoss_vulnerabilities_pb2.py | 4 +- .../v2/scanoss_vulnerabilities_pb2_grpc.py | 12 ++ 11 files changed, 359 insertions(+), 14 deletions(-) create mode 100644 src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py create mode 100644 src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index 56fee94b..23546c71 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) @@ -21,12 +21,16 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _STATUSCODE._serialized_start=220 - _STATUSCODE._serialized_end=316 + _STATUSCODE._serialized_start=336 + _STATUSCODE._serialized_end=432 _STATUSRESPONSE._serialized_start=69 _STATUSRESPONSE._serialized_end=153 _ECHOREQUEST._serialized_start=155 _ECHOREQUEST._serialized_end=185 _ECHORESPONSE._serialized_start=187 _ECHORESPONSE._serialized_end=218 + _PURLREQUEST._serialized_start=220 + _PURLREQUEST._serialized_end=334 + _PURLREQUEST_PURLS._serialized_start=292 + _PURLREQUEST_PURLS._serialized_end=334 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index 8ae503da..ced1fc75 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -14,7 +14,7 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xc5\x02\n\nComponents\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12o\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\x12u\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponseB7Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xcb\x02\n\nComponents\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x12q\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"\x00\x12w\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"\x00\x42\x37Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2b\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', globals()) @@ -39,5 +39,5 @@ _COMPVERSIONRESPONSE_COMPONENT._serialized_start=850 _COMPVERSIONRESPONSE_COMPONENT._serialized_end=981 _COMPONENTS._serialized_start=984 - _COMPONENTS._serialized_end=1309 + _COMPONENTS._serialized_end=1315 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py index c2f1e84d..956d55d4 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py @@ -41,6 +41,10 @@ class ComponentsServicer(object): def Echo(self, request, context): """Standard echo + option (google.api.http) = { + post: "/api/v2/components/echo" + body: "*" + }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -48,6 +52,10 @@ def Echo(self, request, context): def SearchComponents(self, request, context): """Search for components + option (google.api.http) = { + post: "/api/v2/components/search" + body: "*" + }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -55,6 +63,10 @@ def SearchComponents(self, request, context): def GetComponentVersions(self, request, context): """Get all version information for a specific component + option (google.api.http) = { + post: "/api/v2/components/versions" + body: "*" + }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py new file mode 100644 index 00000000..2a84b5e6 --- /dev/null +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: scanoss/api/cryptography/v2/scanoss-cryptography.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xb9\x02\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x31\n\nAlgorithms\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\x1au\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12M\n\nalgorithms\x18\x03 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.AlgorithmResponse.Algorithms2\xc8\x01\n\x0c\x43ryptography\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x12\x65\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"\x00\x42;Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2b\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2' + _ALGORITHMRESPONSE._serialized_start=132 + _ALGORITHMRESPONSE._serialized_end=445 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_start=277 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_end=326 + _ALGORITHMRESPONSE_PURLS._serialized_start=328 + _ALGORITHMRESPONSE_PURLS._serialized_end=445 + _CRYPTOGRAPHY._serialized_start=448 + _CRYPTOGRAPHY._serialized_end=648 +# @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py new file mode 100644 index 00000000..18e23658 --- /dev/null +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py @@ -0,0 +1,194 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from scanoss.api.cryptography.v2 import scanoss_cryptography_pb2 as scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2 + + +class CryptographyStub(object): + """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "SCANOSS Cryptography Service"; + version: "2.0"; + contact: { + name: "scanoss-cryptography"; + url: "https://github.com/scanoss/crpytography"; + email: "support@scanoss.com"; + }; + }; + schemes: HTTP; + consumes: "application/json"; + produces: "application/json"; + responses: { + key: "404"; + value: { + description: "Returned when the resource does not exist."; + schema: { + json_schema: { + type: STRING; + } + } + } + } + }; + + + Expose all of the SCANOSS Cryptography RPCs here + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Echo = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) + self.GetAlgorithms = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, + ) + + +class CryptographyServicer(object): + """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "SCANOSS Cryptography Service"; + version: "2.0"; + contact: { + name: "scanoss-cryptography"; + url: "https://github.com/scanoss/crpytography"; + email: "support@scanoss.com"; + }; + }; + schemes: HTTP; + consumes: "application/json"; + produces: "application/json"; + responses: { + key: "404"; + value: { + description: "Returned when the resource does not exist."; + schema: { + json_schema: { + type: STRING; + } + } + } + } + }; + + + Expose all of the SCANOSS Cryptography RPCs here + """ + + def Echo(self, request, context): + """Standard echo + option (google.api.http) = { + post: "/api/v2/cryptography/echo" + body: "*" + }; + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetAlgorithms(self, request, context): + """Get Cryptographic algorithms associated with a list of PURLs + option (google.api.http) = { + post: "/api/v2/cryptography/algorithms" + body: "*" + }; + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_CryptographyServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetAlgorithms': grpc.unary_unary_rpc_method_handler( + servicer.GetAlgorithms, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Cryptography(object): + """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "SCANOSS Cryptography Service"; + version: "2.0"; + contact: { + name: "scanoss-cryptography"; + url: "https://github.com/scanoss/crpytography"; + email: "support@scanoss.com"; + }; + }; + schemes: HTTP; + consumes: "application/json"; + produces: "application/json"; + responses: { + key: "404"; + value: { + description: "Returned when the resource does not exist."; + schema: { + json_schema: { + type: STRING; + } + } + } + } + }; + + + Expose all of the SCANOSS Cryptography RPCs here + """ + + @staticmethod + def Echo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/Echo', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetAlgorithms(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index ac983c0e..b0613fe8 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -14,7 +14,7 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xd3\x01\n\x0c\x44\x65pendencies\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\x12r\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponseB;Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xd7\x01\n\x0c\x44\x65pendencies\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x12t\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\"\x00\x42;Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2b\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) @@ -37,5 +37,5 @@ _DEPENDENCYRESPONSE_FILES._serialized_start=777 _DEPENDENCYRESPONSE_FILES._serialized_end=910 _DEPENDENCIES._serialized_start=913 - _DEPENDENCIES._serialized_end=1124 + _DEPENDENCIES._serialized_end=1128 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py index cddf7cfa..25c631ff 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py @@ -7,7 +7,33 @@ class DependenciesStub(object): - """ + """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "SCANOSS Dependency Service"; + version: "2.0"; + contact: { + name: "scanoss-dependencies"; + url: "https://github.com/scanoss/dependencies"; + email: "support@scanoss.com"; + }; + }; + schemes: HTTP; + consumes: "application/json"; + produces: "application/json"; + responses: { + key: "404"; + value: { + description: "Returned when the resource does not exist."; + schema: { + json_schema: { + type: STRING; + } + } + } + } + }; + + Expose all of the SCANOSS Dependency RPCs here """ @@ -30,12 +56,42 @@ def __init__(self, channel): class DependenciesServicer(object): - """ + """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "SCANOSS Dependency Service"; + version: "2.0"; + contact: { + name: "scanoss-dependencies"; + url: "https://github.com/scanoss/dependencies"; + email: "support@scanoss.com"; + }; + }; + schemes: HTTP; + consumes: "application/json"; + produces: "application/json"; + responses: { + key: "404"; + value: { + description: "Returned when the resource does not exist."; + schema: { + json_schema: { + type: STRING; + } + } + } + } + }; + + Expose all of the SCANOSS Dependency RPCs here """ def Echo(self, request, context): """Standard echo + option (google.api.http) = { + post: "/api/v2/dependencies/echo" + body: "*" + }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -43,6 +99,10 @@ def Echo(self, request, context): def GetDependencies(self, request, context): """Get dependency details + option (google.api.http) = { + post: "/api/v2/dependencies/dependencies" + body: "*" + }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -69,7 +129,33 @@ def add_DependenciesServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class Dependencies(object): - """ + """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "SCANOSS Dependency Service"; + version: "2.0"; + contact: { + name: "scanoss-dependencies"; + url: "https://github.com/scanoss/dependencies"; + email: "support@scanoss.com"; + }; + }; + schemes: HTTP; + consumes: "application/json"; + produces: "application/json"; + responses: { + key: "404"; + value: { + description: "Returned when the resource does not exist."; + schema: { + json_schema: { + type: STRING; + } + } + } + } + }; + + Expose all of the SCANOSS Dependency RPCs here """ diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index c44a5ad5..0ccff734 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -14,7 +14,7 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto2[\n\x08Scanning\x12O\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponseB3Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto2]\n\x08Scanning\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x42\x33Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2b\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) @@ -23,5 +23,5 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2' _SCANNING._serialized_start=119 - _SCANNING._serialized_end=210 + _SCANNING._serialized_end=212 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index f6530e94..864c1bd6 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -28,6 +28,10 @@ class ScanningServicer(object): def Echo(self, request, context): """Standard echo + option (google.api.http) = { + post: "/api/v2/scanning/echo" + body: "*" + }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index 53326b2b..c041a460 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -14,7 +14,7 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n Date: Sat, 18 Mar 2023 15:05:11 +0000 Subject: [PATCH 104/489] first pass at comp crypto support --- src/scanoss/cli.py | 131 ++++++++++++++++++++++++++++++------- src/scanoss/scanner.py | 65 +++++++++--------- src/scanoss/scanossgrpc.py | 61 ++++++++++++++++- 3 files changed, 201 insertions(+), 56 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index a3942dfc..7660ce62 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ SPDX-License-Identifier: MIT @@ -61,11 +60,6 @@ def setup_args() -> None: description=f'Version of SCANOSS CLI: {__version__}', help='SCANOSS version') p_ver.set_defaults(func=ver) - # Sub-command: fast - p_ver = subparsers.add_parser('fast', - description=f'Is fast winnowing enabled: {__version__}', help='SCANOSS fast winnowing') - p_ver.set_defaults(func=fast) - # Sub-command: scan p_scan = subparsers.add_parser('scan', aliases=['sc'], description=f'Analyse/scan the given source base: {__version__}', @@ -162,13 +156,36 @@ def setup_args() -> None: p_cnv.add_argument('--input-format', type=str, choices=['plain'], default='plain', help='Input format (optional - default: plain)') + # Sub-command: component + p_comp = subparsers.add_parser('component', aliases=['comp'], + description=f'SCANOSS Component commands: {__version__}', + help='Component support commands') + + comp_sub = p_comp.add_subparsers(title='Component Commands', dest='subparsercmd', description='utils sub-commands', + help='component sub-commands') + + # Component Sub-command: component crypto + c_crypto = comp_sub.add_parser('crypto', aliases=['cr'], + description=f'Show Cryptographic algorithms: {__version__}', + help='Retreive the cryptographic algorithms for the given components') + c_crypto.set_defaults(func=comp_crypto) + + c_crypto.add_argument('--purl', '-p', type=str, nargs="*", help='Package URL - PURL to process.') + c_crypto.add_argument('--input', '-i', type=str, help='Input file name') + c_crypto.add_argument('--output','-o', type=str, help='Output result file name (optional - default stdout).') + # Sub-command: utils - p_util = subparsers.add_parser('utils', aliases=['ut', 'util'], + p_util = subparsers.add_parser('utils', aliases=['ut'], description=f'SCANOSS Utility commands: {__version__}', help='General utility support commands') - utils_sub = p_util.add_subparsers(title='Utils Commands', dest='utilsubparser', description='utils sub-commands', - help='utils sub-commands') + utils_sub = p_util.add_subparsers(title='Utils Commands', dest='subparsercmd', description='component sub-commands', + help='component sub-commands') + + # Utils Sub-command: utils fast + p_f_f = utils_sub.add_parser('fast', + description=f'Is fast winnowing enabled: {__version__}', help='SCANOSS fast winnowing') + p_f_f.set_defaults(func=fast) # Utils Sub-command: utils certloc p_c_loc = utils_sub.add_parser('certloc', aliases=['cl'], @@ -197,24 +214,22 @@ def setup_args() -> None: p_p_proxy.add_argument('--url', required=False, type=str, default="https://osskb.org/api", help='URL to test (default: https://osskb.org/api).') - # Global command options + # Global Scan command options for p in [p_scan]: - p.add_argument('--key', '-k', type=str, - help='SCANOSS API Key token (optional - not required for default OSSKB URL)' - ) p.add_argument('--apiurl', type=str, help='SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)' ) - p.add_argument('--api2url', type=str, - help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' + p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') + + # Global Scan/GRPC options + for p in [p_scan, c_crypto]: + p.add_argument('--key', '-k', type=str, + help='SCANOSS API Key token (optional - not required for default OSSKB URL)' ) p.add_argument('--proxy', type=str, help='Proxy URL to use for connections (optional). ' 'Can also use the environment variable "HTTPS_PROXY=:" ' 'and "grcp_proxy=:" for gRPC' ) - p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' - 'Can also use the environment variable "grcp_proxy=:"' - ) p.add_argument('--pac', type=str, help='Proxy auto configuration (optional). ' 'Specify a file, http url or "auto" to try to discover it.' ) @@ -223,9 +238,18 @@ def setup_args() -> None: '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem" and ' '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC' ) - p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') - for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy]: + # Global GRPC options + for p in [p_scan, c_crypto]: + p.add_argument('--api2url', type=str, + help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' + ) + p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' + 'Can also use the environment variable "grcp_proxy=:"' + ) + + # Help/Trace command options + for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -237,9 +261,12 @@ def setup_args() -> None: if not args.subparser: parser.print_help() # No sub command subcommand, print general help exit(1) - elif args.subparser == 'utils' and not args.utilsubparser: # No utils sub command supplied - parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed - exit(1) + else: + if (args.subparser == 'utils' or args.subparser == 'ut' or + args.subparser == 'component' or args.subparser == 'comp') \ + and not args.subparsercmd: + parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed + exit(1) args.func(parser, args) # Execute the function associated with the sub-command def ver(*_): @@ -650,6 +677,64 @@ def get_pac_file(pac: str): return pac_file +def comp_crypto(parser, args): + """ + Run the "component crypto" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + from .scanossgrpc import ScanossGrpc + import json + + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + exit(1) + pac_file = get_pac_file(args.pac) + file = sys.stdout + if args.output: + file = open(args.output, 'w') + success = False + purl_request = {} + if args.input: + if not os.path.isfile(args.input): + print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') + exit(1) + with open(args.input, 'r') as f: + try: + purl_request = json.loads(f.read()) + except Exception as e: + print_stderr(f'ERROR: Problem parsing input JSON: {e}') + elif args.purl: + purls = [] + for p in args.purl: + purls.append({'purl': p}) + purl_request = {'purls': purls} + + purl_count = len(purl_request.get('purls')) + if purl_count == 0: + print_stderr('No PURLs supplied to get cryptographic algorithms.') + + grpc_api = ScanossGrpc(url=args.api2url, debug=args.debug, trace=args.trace, quiet=args.quiet, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file + ) + resp = grpc_api.get_crypto_json(purl_request) + # print_stderr(f'Crypto Algo Resp: {resp}') + if resp: + print(json.dumps(resp, indent=2, sort_keys=True), file=file) + success = True + if args.output: + file.close() + if not success: + exit(1) + def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index c3983371..486a5710 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -41,9 +41,11 @@ from .scanossgrpc import ScanossGrpc from .scantype import ScanType from .scanossbase import ScanossBase + FAST_WINNOWING = False try: from scanoss_winnowing.winnowing import Winnowing + FAST_WINNOWING = True except ModuleNotFoundError or ImportError: FAST_WINNOWING = False @@ -91,6 +93,7 @@ class Scanner(ScanossBase): SCANOSS scanning class Handle the scanning of files, snippets and dependencies """ + def __init__(self, wfp: str = None, scan_output: str = None, output_format: str = 'plain', debug: bool = False, trace: bool = False, quiet: bool = False, api_key: str = None, url: str = None, sbom_path: str = None, scan_type: str = None, flags: str = None, nb_threads: int = 5, @@ -114,7 +117,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.hidden_files_folders = hidden_files_folders self.scan_options = scan_options self._skip_snippets = True if not scan_options & ScanType.SCAN_SNIPPETS.value else False - ver_details = self.__version_details() + ver_details = Scanner.version_details() self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, all_extensions=all_extensions, obfuscate=obfuscate @@ -138,7 +141,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.threaded_scan = None self.max_post_size = post_size * 1024 if post_size > 0 else MAX_POST_SIZE # Set the max post size (default 64k) if self._skip_snippets: - self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets + self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets def __filter_files(self, files: list) -> list: """ @@ -153,10 +156,10 @@ def __filter_files(self, files: list) -> list: ignore = True if not ignore and not self.all_extensions: # Skip this check if we're allowing all extensions f_lower = f.lower() - if f_lower in FILTERED_FILES: # Check for exact files to ignore + if f_lower in FILTERED_FILES: # Check for exact files to ignore ignore = True if not ignore: - for ending in FILTERED_EXT: # Check for file endings to ignore + for ending in FILTERED_EXT: # Check for file endings to ignore if f_lower.endswith(ending): ignore = True break @@ -177,10 +180,10 @@ def __filter_dirs(self, dirs: list) -> list: ignore = True if not ignore and not self.all_folders: # Skip this check if we're allowing all folders d_lower = d.lower() - if d_lower in FILTERED_DIRS: # Ignore specific folders + if d_lower in FILTERED_DIRS: # Ignore specific folders ignore = True if not ignore: - for de in FILTERED_DIR_EXT: # Ignore specific folder endings + for de in FILTERED_DIR_EXT: # Ignore specific folder endings if d_lower.endswith(de): ignore = True break @@ -244,7 +247,7 @@ def valid_json_file(json_file: str) -> bool: return True @staticmethod - def __version_details() -> str: + def version_details() -> str: """ Extract the date this version was produced :return: version creation date string @@ -351,7 +354,7 @@ def scan_folder(self, scan_dir: str) -> bool: if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f"ERROR: Specified folder does not exist or is not a folder: {scan_dir}") - scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir)+1 + scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir) + 1 self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None if not self.quiet and self.isatty: @@ -367,17 +370,18 @@ def scan_folder(self, scan_dir: str) -> bool: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break - dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories - filtered_files = self.__filter_files(files) # Strip out unwanted files + dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories + filtered_files = self.__filter_files(files) # Strip out unwanted files self.print_debug(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}') - for file in filtered_files: # Cycle through each filtered file + for file in filtered_files: # Cycle through each filtered file path = os.path.join(root, file) f_size = 0 try: f_size = os.stat(path).st_size except Exception as e: - self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink - if f_size > 0: # Ignore broken links and empty files + self.print_trace( + f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink + if f_size > 0: # Ignore broken links and empty files self.print_trace(f'Fingerprinting {path}...') if spinner: spinner.next() @@ -434,7 +438,7 @@ def __run_scan_threaded(self, scan_started: bool, file_count: int) -> bool: success = True self.threaded_scan.update_bar(create=True, file_count=file_count) if not scan_started: - if not self.threaded_scan.run(wait=False): # Run the scan but do not wait for it to complete + if not self.threaded_scan.run(wait=False): # Run the scan but do not wait for it to complete self.print_stderr(f'Warning: Some errors encounted while scanning. Results might be incomplete.') success = False return success @@ -449,7 +453,7 @@ def __finish_scan_threaded(self, file_map: dict = None) -> bool: responses = None dep_responses = None if self.is_file_or_snippet_scan(): - if not self.threaded_scan.complete(): # Wait for the scans to complete + if not self.threaded_scan.complete(): # Wait for the scans to complete self.print_stderr(f'Warning: Scanning analysis ran into some trouble.') success = False self.threaded_scan.complete_bar() @@ -590,7 +594,7 @@ def scan_wfp_file(self, file: str = None) -> bool: :return: True if successful, False otherwise """ success = True - wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default + wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") file_count = Scanner.__count_files_in_wfp_file(wfp_file) @@ -611,13 +615,13 @@ def scan_wfp_file(self, file: str = None) -> bool: for line in f: if line.startswith(WFP_FILE_START): if file_print: - wfp += file_print # Store the WFP for the current file + wfp += file_print # Store the WFP for the current file cur_size = len(wfp.encode("utf-8")) - file_print = line # Start storing the next file + file_print = line # Start storing the next file cur_files += 1 batch_files += 1 else: - file_print += line # Store the rest of the WFP for this file + file_print += line # Store the rest of the WFP for this file l_size = cur_size + len(file_print.encode('utf-8')) # Hit the max post size, so sending the current batch and continue processing if l_size >= self.max_post_size and wfp: @@ -691,7 +695,7 @@ def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> boo return: True if successful, False otherwise """ success = True - wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default + wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") cur_size = 0 @@ -700,16 +704,16 @@ def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> boo scan_started = False wfp = '' scan_block = '' - with open(wfp_file) as f: # Parse the WFP file + with open(wfp_file) as f: # Parse the WFP file for line in f: if line.startswith(WFP_FILE_START): if scan_block: - wfp += scan_block # Store the WFP for the current file + wfp += scan_block # Store the WFP for the current file cur_size = len(wfp.encode("utf-8")) - scan_block = line # Start storing the next file + scan_block = line # Start storing the next file file_count += 1 else: - scan_block += line # Store the rest of the WFP for this file + scan_block += line # Store the rest of the WFP for this file l_size = cur_size + len(scan_block.encode('utf-8')) # Hit the max post size, so sending the current batch and continue processing if l_size >= self.max_post_size and wfp: @@ -722,7 +726,7 @@ def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> boo scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( - f'Warning: Some errors encounted while scanning. Results might be incomplete.') + f'Warning: Some errors encounted while scanning. Results might be incomplete.') success = False # End for loop if scan_block: @@ -803,14 +807,14 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f"ERROR: Specified folder does not exist or is not a folder: {scan_dir}") wfps = '' - scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir)+1 + scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir) + 1 self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') for root, dirs, files in os.walk(scan_dir): - dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories - filtered_files = self.__filter_files(files) # Strip out unwanted files + dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories + filtered_files = self.__filter_files(files) # Strip out unwanted files self.print_trace(f'Root: {root}, Dirs: {dirs}, Files {filtered_files}') for file in filtered_files: path = os.path.join(root, file) @@ -818,8 +822,9 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): try: f_size = os.stat(path).st_size except Exception as e: - self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink - if f_size > 0: # Ignore empty files + self.print_trace( + f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink + if f_size > 0: # Ignore empty files self.print_debug(f'Fingerprinting {path}...') if spinner: spinner.next() diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 54a5ed45..9de43c3c 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -33,9 +33,11 @@ from pypac.resolver import ProxyResolver from urllib.parse import urlparse +from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub +from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, DependencyResponse -from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode +from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest from .scanossbase import ScanossBase from . import __version__ @@ -95,14 +97,16 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu cert_data = ScanossGrpc._load_cert(ca_cert) self.print_debug(f'Setting up (secure: {secure}) connection to {self.url}...') self._get_proxy_config() - if secure is False: - self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) # insecure connection + if secure is False: # insecure connection + self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) + self.crypto_stub = CryptographyStub(grpc.insecure_channel(self.url)) else: if ca_cert is not None: credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate else: credentials = grpc.ssl_channel_credentials() # secure connection with default certificate self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) + self.crypto_stub = CryptographyStub(grpc.secure_channel(self.url, credentials)) @classmethod def _load_cert(cls, cert_file: str) -> bytes: @@ -142,6 +146,28 @@ def deps_echo(self, message: str = 'Hello there!') -> str: self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}') return None + def crypto_echo(self, message: str = 'Hello there!') -> str: + """ + Send Echo message to the Cryptography service + :param self: + :param message: Message to send (default: Hello there!) + :return: echo or None + """ + request_id = str(uuid.uuid4()) + resp: EchoResponse + try: + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + resp = self.crypto_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3) + except Exception as e: + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') + else: + if resp: + return resp.message + self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}') + return None + def get_dependencies(self, dependencies: json, depth: int = 1) -> dict: if not dependencies: self.print_stderr('ERROR: No dependency data supplied to submit to the API.') @@ -184,6 +210,35 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: return MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dictionary return None + def get_crypto_json(self, purls: dict) -> dict: + """ + Client function to call the rpc for Cryptography GetAlgorithms + :param purls: Message to send to the service + :return: Server response or None + """ + if not purls: + self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + return None + request_id = str(uuid.uuid4()) + resp: AlgorithmResponse + try: + request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...') + resp = self.crypto_stub.GetAlgorithms(request, metadata=metadata, timeout=600) + except Exception as e: + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') + else: + if resp: + if not self._check_status_response(resp.status, request_id): + return None + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dictionary + del resp_dict['status'] + return resp_dict + return None + def _check_status_response(self, status_response: StatusResponse, request_id: str = None) -> bool: """ Check the response object to see if the command was successful or not From f1f90a35c43a7d12b5c165811d0d26c9ed76d681 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Sat, 18 Mar 2023 15:53:29 +0000 Subject: [PATCH 105/489] added timeout parameter --- src/scanoss/scanossgrpc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 9de43c3c..23f1daad 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -53,7 +53,7 @@ class ScanossGrpc(ScanossBase): """ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - ca_cert: str = None, api_key: str = None, ver_details: str = None, + ca_cert: str = None, api_key: str = None, ver_details: str = None, timeout: int = 600, proxy: str = None, grpc_proxy: str = None, pac: PACFile = None): """ @@ -75,6 +75,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu self.url = self.url.lower() self.orig_url = self.url # Used for proxy lookup self.api_key = api_key if api_key else SCANOSS_API_KEY + self.timeout = timeout self.proxy = proxy self.grpc_proxy = grpc_proxy self.pac = pac @@ -199,7 +200,7 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...') - resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=600) + resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout) except Exception as e: self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' f'(rqId: {request_id}): {e}') @@ -226,7 +227,7 @@ def get_crypto_json(self, purls: dict) -> dict: metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...') - resp = self.crypto_stub.GetAlgorithms(request, metadata=metadata, timeout=600) + resp = self.crypto_stub.GetAlgorithms(request, metadata=metadata, timeout=self.timeout) except Exception as e: self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' f'(rqId: {request_id}): {e}') From 2c5566bbf2aa6f85fd32971674d2570ab45e0232 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 21 Mar 2023 09:44:32 +0000 Subject: [PATCH 106/489] added official support for cryptographic reporting --- .github/workflows/python-local-test.yml | 4 +- .github/workflows/python-publish-pypi.yml | 6 +- .github/workflows/python-publish-testpypi.yml | 4 +- CHANGELOG.md | 6 + src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 46 +----- src/scanoss/components.py | 150 ++++++++++++++++++ 7 files changed, 171 insertions(+), 47 deletions(-) create mode 100644 src/scanoss/components.py diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 5d5bc793..6a375ca5 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -42,7 +42,7 @@ jobs: run: | which scanoss-py scanoss-py version - scanoss-py fast + scanoss-py utils fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" @@ -56,7 +56,7 @@ jobs: pip install scanoss_winnowing which scanoss-py scanoss-py version - scanoss-py fast + scanoss-py utils fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index a23cf80e..5ef8663a 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -36,7 +36,7 @@ jobs: run: | which scanoss-py scanoss-py version - scanoss-py fast + scanoss-py utils fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" @@ -85,7 +85,7 @@ jobs: run: | which scanoss-py scanoss-py version - scanoss-py fast + scanoss-py utils fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" @@ -99,7 +99,7 @@ jobs: pip install scanoss_winnowing which scanoss-py scanoss-py version - scanoss-py fast + scanoss-py utils fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index f6eb6419..9061225c 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -79,7 +79,7 @@ jobs: run: | which scanoss-py scanoss-py version - scanoss-py fast + scanoss-py utils fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" @@ -93,7 +93,7 @@ jobs: pip install scanoss_winnowing which scanoss-py scanoss-py version - scanoss-py fast + scanoss-py utils fast scanoss-py scan tests > results.json id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" diff --git a/CHANGELOG.md b/CHANGELOG.md index 127ef14f..95b6f73e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.5.0] - 2023-03-21 +### Added +- Added support for component cryptographic reporting + - `scanoss-py component crypto ...` + ## [1.4.2] - 2023-03-09 ### Fixed - Fixed issue with custom certificate when scanning (--ca-cert) @@ -218,3 +223,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.3.7]: https://github.com/scanoss/scanoss.py/compare/v1.3.6...v1.3.7 [1.4.0]: https://github.com/scanoss/scanoss.py/compare/v1.3.7...v1.4.0 [1.4.2]: https://github.com/scanoss/scanoss.py/compare/v1.4.0...v1.4.2 +[1.5.0]: https://github.com/scanoss/scanoss.py/compare/v1.4.2...v1.5.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index ac053c5f..afda04e3 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.4.2' +__version__ = '1.5.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 7660ce62..642bb1e2 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -34,6 +34,7 @@ from .cyclonedx import CycloneDx from .spdxlite import SpdxLite from .csvoutput import CsvOutput +from .components import Components from . import __version__ from .scanner import FAST_WINNOWING @@ -173,6 +174,8 @@ def setup_args() -> None: c_crypto.add_argument('--purl', '-p', type=str, nargs="*", help='Package URL - PURL to process.') c_crypto.add_argument('--input', '-i', type=str, help='Input file name') c_crypto.add_argument('--output','-o', type=str, help='Output result file name (optional - default stdout).') + c_crypto.add_argument('--timeout', '-M', type=int, default=600, + help='Timeout (in seconds) for API communication (optional - default 600)') # Sub-command: utils p_util = subparsers.add_parser('utils', aliases=['ut'], @@ -687,9 +690,6 @@ def comp_crypto(parser, args): args: Namespace Parsed arguments """ - from .scanossgrpc import ScanossGrpc - import json - if (not args.purl and not args.input) or (args.purl and args.input): print_stderr('Please specify an input file or purl to decorate (--purl or --input)') parser.parse_args([args.subparser, args.subparsercmd, '-h']) @@ -698,43 +698,11 @@ def comp_crypto(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) pac_file = get_pac_file(args.pac) - file = sys.stdout - if args.output: - file = open(args.output, 'w') - success = False - purl_request = {} - if args.input: - if not os.path.isfile(args.input): - print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') - exit(1) - with open(args.input, 'r') as f: - try: - purl_request = json.loads(f.read()) - except Exception as e: - print_stderr(f'ERROR: Problem parsing input JSON: {e}') - elif args.purl: - purls = [] - for p in args.purl: - purls.append({'purl': p}) - purl_request = {'purls': purls} - - purl_count = len(purl_request.get('purls')) - if purl_count == 0: - print_stderr('No PURLs supplied to get cryptographic algorithms.') - - grpc_api = ScanossGrpc(url=args.api2url, debug=args.debug, trace=args.trace, quiet=args.quiet, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file - ) - resp = grpc_api.get_crypto_json(purl_request) - # print_stderr(f'Crypto Algo Resp: {resp}') - if resp: - print(json.dumps(resp, indent=2, sort_keys=True), file=file) - success = True - if args.output: - file.close() - if not success: + comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + timeout=args.timeout) + if not comps.get_crypto_details(args.input, args.purl, args.output): exit(1) - def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/components.py b/src/scanoss/components.py new file mode 100644 index 00000000..6efe338f --- /dev/null +++ b/src/scanoss/components.py @@ -0,0 +1,150 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2023, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +import os +import sys +from typing import TextIO + +from pypac.parser import PACFile + +from .scanner import Scanner +from .scanossbase import ScanossBase +from .scanossgrpc import ScanossGrpc + + +class Components(ScanossBase): + """ + Class for Component functionality + """ + + def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False, + grpc_url: str = None, api_key: str = None, timeout: int = 600, + proxy: str = None, grpc_proxy: str = None, ca_cert: str = None, pac: PACFile = None + ): + """ + Handle all component style requests + + :param debug: Debug + :param trace: Trace + :param quiet: Quiet + :param grpc_url: gRPC URL + :param api_key: API Key + :param timeout: Timeout for requests (default 600) + :param proxy: Proxy to use (optional) + :param grpc_proxy: Specific gRPC proxy (optional) + :param ca_cert: TLS client certificate (optional) + :param pac: Proxy Auto-Config file (optional) + """ + super().__init__(debug, trace, quiet) + ver_details = Scanner.version_details() + self.grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, + ver_details=ver_details, ca_cert=ca_cert, proxy=proxy, pac=pac, + grpc_proxy=grpc_proxy, timeout=timeout) + + def load_purls(self, json_file: str = None, purls: [] = None) -> dict: + """ + Load the specified purls and return a dictionary + + :param json_file: JSON PURL file (optional) + :param purls: list of PURLs (optional) + :return: PURL Request dictionary + """ + if json_file: + if not os.path.isfile(json_file) or not os.access(json_file, os.R_OK): + self.print_stderr(f'ERROR: JSON file does not exist, is not a file, or is not readable: {json_file}') + return None + with open(json_file, 'r') as f: + try: + purl_request = json.loads(f.read()) + except Exception as e: + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + return None + elif purls: + parsed_purls = [] + for p in purls: + parsed_purls.append({'purl': p}) + purl_request = {'purls': parsed_purls} + else: + self.print_stderr('ERROR: No purls specified to process.') + return None + purl_count = len(purl_request.get('purls', [])) + self.print_debug(f'Parsed Purls ({purl_count}): {purl_request}') + if purl_count == 0: + self.print_stderr('ERROR: No PURLs parsed from request.') + return None + return purl_request + + def _open_file_or_sdtout(self, filename): + """ + Open the given filename if requested, otherwise return STDOUT + + :param filename: + :return: + """ + file = sys.stdout + if filename: + try: + file = open(filename, 'w') + except OSError as e: + self.print_stderr(f'ERROR: Failed to open output file {filename}: {e}') + return None + return file + + def _close_file(self, filename: str = None, file: TextIO = None) -> None: + """ + Close the file descriptor if its defined + + :param filename: filename + :param file: file IO object + :return: None + """ + if filename and file: + self.print_trace(f'Closing file: {filename}') + file.close() + + def get_crypto_details(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: + """ + Retrieve the cryptographic details for the supplied PURLs + + :param json_file: PURL JSON request file (optional) + :param purls: PURL request array (optional) + :param output_file: output filename (optional). Default: STDOUT + :return: True on success, False otherwise + """ + success = False + purls_request = self.load_purls(json_file, purls) + if purls_request is None or len(purls_request) == 0: + return False + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + self.print_msg('Sending PURLs to Crypto API for decoration...') + response = self.grpc_api.get_crypto_json(purls_request) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success From a176f96715db24048d00dc8b13bc4f47456796fd Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 21 Mar 2023 09:50:49 +0000 Subject: [PATCH 107/489] moved fast command to utils fast --- .github/workflows/container-local-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index cf112a21..ecb61c49 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -53,7 +53,7 @@ jobs: docker load --input /tmp/scanoss-py.tar docker image ls -a docker run ${{ env.IMAGE_NAME }} version - docker run ${{ env.IMAGE_NAME }} fast + docker run ${{ env.IMAGE_NAME }} utils fast docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" From 57e39a36bb40cc9e77bfb32fc264b9eff1605ff6 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 21 Apr 2023 12:56:10 +0100 Subject: [PATCH 108/489] added support for STDIN input when scanning or fingerprinting --- CHANGELOG.md | 7 ++++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 41 +++++++++++++++++++++--------- src/scanoss/scanner.py | 55 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95b6f73e..4a8b9c35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.5.1] - 2023-04-21 +### Added +- Added support scanning/fingeprinting file contents from STDIN + - `cat test.py | scanoss-py scan --stdin test.py -o results.json` + - `cat test.py | scanoss-py wfp --stdin test.py -o fingers.wfp` + ## [1.5.0] - 2023-03-21 ### Added - Added support for component cryptographic reporting @@ -224,3 +230,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.4.0]: https://github.com/scanoss/scanoss.py/compare/v1.3.7...v1.4.0 [1.4.2]: https://github.com/scanoss/scanoss.py/compare/v1.4.0...v1.4.2 [1.5.0]: https://github.com/scanoss/scanoss.py/compare/v1.4.2...v1.5.0 +[1.5.1]: https://github.com/scanoss/scanoss.py/compare/v1.5.0...v1.5.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index afda04e3..d9ff6931 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.5.0' +__version__ = '1.5.1' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 642bb1e2..86ed11d7 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -73,6 +73,9 @@ def setup_args() -> None: p_scan.add_argument('--dep', '-p', type=str, help='Use a dependency file instead of a folder (optional)' ) + p_scan.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, + help='Scan the file contents supplied via STDIN (optional)' + ) p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file') p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file') p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -117,6 +120,9 @@ def setup_args() -> None: p_wfp.set_defaults(func=wfp) p_wfp.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') + p_wfp.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, + help='Fingerprint the file contents supplied via STDIN (optional)' + ) p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') @@ -328,8 +334,8 @@ def wfp(parser, args): args: Namespace Parsed arguments """ - if not args.scan_dir: - print_stderr('Please specify a file/folder') + if not args.scan_dir and not args.stdin: + print_stderr('Please specify a file/folder or STDIN (--stdin)') parser.parse_args([args.subparser, '-h']) exit(1) scan_output: str = None @@ -342,15 +348,22 @@ def wfp(parser, args): scan_options=scan_options, all_extensions=args.all_extensions, all_folders=args.all_folders, hidden_files_folders=args.all_hidden) - if not os.path.exists(args.scan_dir): - print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') - exit(1) - if os.path.isdir(args.scan_dir): - scanner.wfp_folder(args.scan_dir, scan_output) - elif os.path.isfile(args.scan_dir): - scanner.wfp_file(args.scan_dir, scan_output) + if args.stdin: + contents = sys.stdin.buffer.read() + scanner.wfp_contents(args.stdin, contents, scan_output) + elif args.scan_dir: + if not os.path.exists(args.scan_dir): + print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') + exit(1) + if os.path.isdir(args.scan_dir): + scanner.wfp_folder(args.scan_dir, scan_output) + elif os.path.isfile(args.scan_dir): + scanner.wfp_file(args.scan_dir, scan_output) + else: + print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') + exit(1) else: - print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') + print_stderr('No action found to process') exit(1) @@ -396,8 +409,8 @@ def scan(parser, args): args: Namespace Parsed arguments """ - if not args.scan_dir and not args.wfp: - print_stderr('Please specify a file/folder or fingerprint (--wfp)') + if not args.scan_dir and not args.wfp and not args.stdin: + print_stderr('Please specify a file/folder, fingerprint (--wfp) or STDIN (--stdin)') parser.parse_args([args.subparser, '-h']) exit(1) if args.pac and args.proxy: @@ -493,6 +506,10 @@ def scan(parser, args): scanner.scan_wfp_file_threaded(args.wfp) else: scanner.scan_wfp_file(args.wfp) + elif args.stdin: + contents = sys.stdin.buffer.read() + if not scanner.scan_contents(args.stdin, contents): + exit(1) elif args.scan_dir: if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 486a5710..9e61eb9a 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -587,6 +587,35 @@ def scan_file(self, file: str) -> bool: success = False return success + def scan_contents(self, filename: str, contents: bytes) -> bool: + """ + Scan the given contents as a file + + :param filename: filename to associate with the contents + :param contents: file contents + :return: True if successful, False otherwise + """ + success = True + if not filename: + raise Exception(f"ERROR: Please specify a filename to scan") + if not contents: + raise Exception(f"ERROR: Please specify a file contents to scan") + + self.print_debug(f'Fingerprinting {filename}...') + wfp = self.winnowing.wfp_for_contents(filename, False, contents) + if wfp is not None and wfp != '': + if self.threaded_scan: + self.threaded_scan.queue_add(wfp) # Submit the WFP for scanning + self.print_debug(f'Scanning {filename}...') + if self.threaded_scan: + success = self.__run_scan_threaded(False, 1) + else: + success = False + if self.threaded_scan: + if not self.__finish_scan_threaded(): + success = False + return success + def scan_wfp_file(self, file: str = None) -> bool: """ Scan the contents of the specified WFP file (in the current process) @@ -777,6 +806,32 @@ def scan_wfp(self, wfp: str) -> bool: return success + def wfp_contents(self, filename: str, contents: bytes, wfp_file: str = None): + """ + Fingerprint the specified contents as a file + + :param filename: filename to associate with the contents + :param contents: file contents + :param wfp_file: WFP to write results to (optional) + :return: + """ + if not filename: + raise Exception(f"ERROR: Please specify a filename to scan") + if not contents: + raise Exception(f"ERROR: Please specify a file contents to scan") + + self.print_debug(f'Fingerprinting {filename}...') + wfp = self.winnowing.wfp_for_contents(filename, False, contents) + if wfp: + if wfp_file: + self.print_stderr(f'Writing fingerprints to {wfp_file}') + with open(wfp_file, 'w') as f: + f.write(wfp) + else: + print(wfp) + else: + Scanner.print_stderr(f'Warning: No fingerprints generated for: {scan_file}') + def wfp_file(self, scan_file: str, wfp_file: str = None): """ Fingerprint the specified file From d57b7303c4609beae7e01276bf4d9ec95d1b019e Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Fri, 21 Apr 2023 14:34:01 +0100 Subject: [PATCH 109/489] fix fast test command --- .github/workflows/container-publish-ghcr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 5e58dedc..94d5fccf 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -81,7 +81,7 @@ jobs: run: | docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} version - docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} fast + docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} utils fast docker run -v "$(pwd)":"/scanoss" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" From 55a353c59f2887088ce1dc75c28201464598141b Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Mon, 24 Apr 2023 16:05:41 +0100 Subject: [PATCH 110/489] CLI help updates --- src/scanoss/cli.py | 56 ++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 86ed11d7..be1ab659 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -54,8 +54,7 @@ def setup_args() -> None: parser.add_argument('--version', '-v', action='store_true', help='Display version details') subparsers = parser.add_subparsers(title='Sub Commands', dest='subparser', description='valid subcommands', - help='sub-command help' - ) + help='sub-command help') # Sub-command: version p_ver = subparsers.add_parser('version', aliases=['ver'], description=f'Version of SCANOSS CLI: {__version__}', help='SCANOSS version') @@ -68,50 +67,41 @@ def setup_args() -> None: p_scan.set_defaults(func=scan) p_scan.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_scan.add_argument('--wfp', '-w', type=str, - help='Scan a WFP File instead of a folder (optional)' - ) + help='Scan a WFP File instead of a folder (optional)') p_scan.add_argument('--dep', '-p', type=str, - help='Use a dependency file instead of a folder (optional)' - ) + help='Use a dependency file instead of a folder (optional)') p_scan.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, - help='Scan the file contents supplied via STDIN (optional)' - ) + help='Scan the file contents supplied via STDIN (optional)') p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file') p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file') p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_scan.add_argument('--format', '-f', type=str, choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], - help='Result output format (optional - default: plain)' - ) + help='Result output format (optional - default: plain)') p_scan.add_argument('--threads', '-T', type=int, default=10, - help='Number of threads to use while scanning (optional - default 10)' - ) + help='Number of threads to use while scanning (optional - default 10)') p_scan.add_argument('--flags', '-F', type=int, help='Scanning engine flags (1: disable snippet matching, 2 enable snippet ids, ' '4: disable dependencies, 8: disable licenses, 16: disable copyrights,' '32: disable vulnerabilities, 64: disable quality, 128: disable cryptography,' '256: disable best match only, 512: hide identified files, ' '1024: enable download_url, 2048: enable GitHub full path, ' - '4096: disable extended server stats)' - ) + '4096: disable extended server stats)') p_scan.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') p_scan.add_argument('--post-size', '-P', type=int, default=64, - help='Number of kilobytes to limit the post to while scanning (optional - default 64)' - ) + help='Number of kilobytes to limit the post to while scanning (optional - default 64)') p_scan.add_argument('--timeout', '-M', type=int, default=120, - help='Timeout (in seconds) for API communication (optional - default 120)' - ) + help='Timeout (in seconds) for API communication (optional - default 120)') p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') p_scan.add_argument('--all-extensions', action='store_true', help='Scan all file extensions') p_scan.add_argument('--all-folders', action='store_true', help='Scan all folders') p_scan.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') - p_scan.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') + p_scan.add_argument('--obfuscate', action='store_true', help='Obfuscate file paths and names') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') p_scan.add_argument('--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).') p_scan.add_argument('--sc-timeout', type=int, default=600, - help='Timeout (in seconds) for scancode to complete (optional - default 600)' - ) + help='Timeout (in seconds) for scancode to complete (optional - default 600)') # Sub-command: fingerprint p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'], @@ -121,8 +111,7 @@ def setup_args() -> None: p_wfp.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_wfp.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, - help='Fingerprint the file contents supplied via STDIN (optional)' - ) + help='Fingerprint the file contents supplied via STDIN (optional)') p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') @@ -226,36 +215,29 @@ def setup_args() -> None: # Global Scan command options for p in [p_scan]: p.add_argument('--apiurl', type=str, - help='SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)' - ) + help='SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)') p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') # Global Scan/GRPC options for p in [p_scan, c_crypto]: p.add_argument('--key', '-k', type=str, - help='SCANOSS API Key token (optional - not required for default OSSKB URL)' - ) + help='SCANOSS API Key token (optional - not required for default OSSKB URL)') p.add_argument('--proxy', type=str, help='Proxy URL to use for connections (optional). ' 'Can also use the environment variable "HTTPS_PROXY=:" ' - 'and "grcp_proxy=:" for gRPC' - ) + 'and "grcp_proxy=:" for gRPC') p.add_argument('--pac', type=str, help='Proxy auto configuration (optional). ' - 'Specify a file, http url or "auto" to try to discover it.' - ) + 'Specify a file, http url or "auto" to try to discover it.') p.add_argument('--ca-cert', type=str, help='Alternative certificate PEM file (optional). ' 'Can also use the environment variable ' '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem" and ' - '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC' - ) + '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC') # Global GRPC options for p in [p_scan, c_crypto]: p.add_argument('--api2url', type=str, - help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' - ) + help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)') p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' - 'Can also use the environment variable "grcp_proxy=:"' - ) + 'Can also use the environment variable "grcp_proxy=:"') # Help/Trace command options for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto]: From ae28d9be93ddac3b18811104201f28ecaa2bf283 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Tue, 13 Jun 2023 11:41:07 +0100 Subject: [PATCH 111/489] added retry command option for scanning --- CHANGELOG.md | 6 ++++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 8 +++++++- src/scanoss/scanner.py | 4 ++-- src/scanoss/scanossapi.py | 11 ++++++----- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a8b9c35..501385d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.5.2] - 2023-06-13 +### Added +- Added retry limit option (`--retry`) while scanning + - `--retry 0` will fail immediately + ## [1.5.1] - 2023-04-21 ### Added - Added support scanning/fingeprinting file contents from STDIN @@ -231,3 +236,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.4.2]: https://github.com/scanoss/scanoss.py/compare/v1.4.0...v1.4.2 [1.5.0]: https://github.com/scanoss/scanoss.py/compare/v1.4.2...v1.5.0 [1.5.1]: https://github.com/scanoss/scanoss.py/compare/v1.5.0...v1.5.1 +[1.5.2]: https://github.com/scanoss/scanoss.py/compare/v1.5.1...v1.5.2 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index d9ff6931..495162bc 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.5.1' +__version__ = '1.5.2' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index be1ab659..bad0f61c 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -91,6 +91,8 @@ def setup_args() -> None: help='Number of kilobytes to limit the post to while scanning (optional - default 64)') p_scan.add_argument('--timeout', '-M', type=int, default=120, help='Timeout (in seconds) for API communication (optional - default 120)') + p_scan.add_argument('--retry', '-R', type=int, default=5, + help='Retry limit for API communication (optional - default 5)') p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') p_scan.add_argument('--all-extensions', action='store_true', help='Scan all file extensions') p_scan.add_argument('--all-folders', action='store_true', help='Scan all folders') @@ -445,6 +447,8 @@ def scan(parser, args): print_stderr(f'Changing scanning POST size to: {args.post_size}k...') if args.timeout != 120: print_stderr(f'Changing scanning POST timeout to: {args.timeout}...') + if args.retry != 5: + print_stderr(f'Changing scanning POST retry to: {args.retry}...') if args.obfuscate: print_stderr("Obfuscating file fingerprints...") if args.proxy: @@ -460,6 +464,8 @@ def scan(parser, args): elif not args.quiet: if args.timeout < 5: print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') + if args.retry < 0: + print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.') if not os.access(os.getcwd(), os.W_OK): # Make sure the current directory is writable. If not disable saving WFP print_stderr(f'Warning: Current directory is not writable: {os.getcwd()}') @@ -478,7 +484,7 @@ def scan(parser, args): scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, grpc_url=args.api2url, obfuscate=args.obfuscate, ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy, - pac=pac_file, ca_cert=args.ca_cert + pac=pac_file, ca_cert=args.ca_cert, retry=args.retry ) if args.wfp: if not scanner.is_file_or_snippet_scan(): diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 9e61eb9a..4725bbff 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -101,7 +101,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, - ca_cert: str = None, pac: PACFile = None + ca_cert: str = None, pac: PACFile = None, retry: int = 5 ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning @@ -125,7 +125,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, ver_details=ver_details, ignore_cert_errors=ignore_cert_errors, - proxy=proxy, ca_cert=ca_cert, pac=pac + proxy=proxy, ca_cert=ca_cert, pac=pac, retry=retry ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 5d72c3dd..4d803e8d 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -53,7 +53,7 @@ class ScanossApi(ScanossBase): def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None, url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, timeout: int = 120, ver_details: str = None, ignore_cert_errors: bool = False, - proxy: str = None, ca_cert: str = None, pac: PACFile = None): + proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5): """ Initialise the SCANOSS API :param scan_type: Scan type (default identify) @@ -82,6 +82,7 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st self.sbom_path = sbom_path self.flags = flags self.timeout = timeout if timeout > 5 else 120 + self.retry_limit = retry if retry >= 0 else 5 self.ignore_cert_errors = ignore_cert_errors self.headers = {} if ver_details: @@ -149,7 +150,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): headers['x-request-id'] = request_id # send a unique request id for each post r = None retry = 0 # Add some retry logic to cater for timeouts, etc. - while retry <= 5: + while retry <= self.retry_limit: retry += 1 try: r = None @@ -163,7 +164,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: - if retry > 5: # Timed out 5 or more times, fail + if retry > self.retry_limit: # Timed out retry_limit or more times, fail self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data ({request_id}) - {e}: {scan_files}') raise Exception(f"ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for" f" {self.url}") from e @@ -176,7 +177,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e else: if r is None: - if retry > 5: # No response 5 or more times, fail + if retry > self.retry_limit: # No response retry_limit or more times, fail self.save_bad_req_wfp(scan_files, request_id, scan_id) raise Exception(f"ERROR: The SCANOSS API request ({request_id}) response object is empty " f"for {self.url}") @@ -190,7 +191,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): raise Exception(f"ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected " f"for {self.url} due to service limits being exceeded.") elif r.status_code >= 400: - if retry > 5: # No response 5 or more times, fail + if retry > self.retry_limit: # No response retry_limit or more times, fail self.save_bad_req_wfp(scan_files, request_id, scan_id) raise Exception( f"ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, " From 46a655c054cae4a56e0e251c2a7237b7877c11e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez?= Date: Fri, 16 Jun 2023 13:20:11 -0300 Subject: [PATCH 112/489] Adds High Precision Snippet Matching support (#21) --- CHANGELOG.md | 5 ++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 8 +++-- src/scanoss/scanner.py | 5 ++-- src/scanoss/winnowing.py | 64 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 78 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 501385d1..e470d37b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.6.0] - 2023-06-16 +### Added +- Added support for High Precision Snippet Matching (`--hpsm` or `-H`) while scanning + - `scanoss-py scan --hpsm ...` + ## [1.5.2] - 2023-06-13 ### Added - Added retry limit option (`--retry`) while scanning diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 495162bc..3bbcb66e 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.5.2' +__version__ = '1.6.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index bad0f61c..bc9f0a82 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -104,6 +104,7 @@ def setup_args() -> None: help='Scancode command and path if required (optional - default scancode).') p_scan.add_argument('--sc-timeout', type=int, default=600, help='Timeout (in seconds) for scancode to complete (optional - default 600)') + p_scan.add_argument('--hpsm', '-H', action='store_true', help='Scan using High Precision Snippet Matching') # Sub-command: fingerprint p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'], @@ -120,6 +121,7 @@ def setup_args() -> None: p_wfp.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions') p_wfp.add_argument('--all-folders', action='store_true', help='Fingerprint all folders') p_wfp.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders') + p_wfp.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.') # Sub-command: dependency p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], @@ -330,7 +332,7 @@ def wfp(parser, args): scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value # Skip snippet generation or not scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, obfuscate=args.obfuscate, scan_options=scan_options, all_extensions=args.all_extensions, - all_folders=args.all_folders, hidden_files_folders=args.all_hidden) + all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm) if args.stdin: contents = sys.stdin.buffer.read() @@ -459,6 +461,8 @@ def scan(parser, args): print_stderr(f'Using Proxy Auto-config (PAC) {args.pac}...') if args.ca_cert: print_stderr(f'Using Certificate {args.ca_cert}...') + if args.hpsm: + print_stderr("Setting HPSM mode...") if flags: print_stderr(f'Using flags {flags}...') elif not args.quiet: @@ -484,7 +488,7 @@ def scan(parser, args): scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, grpc_url=args.api2url, obfuscate=args.obfuscate, ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy, - pac=pac_file, ca_cert=args.ca_cert, retry=args.retry + pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, hpsm=args.hpsm ) if args.wfp: if not scanner.is_file_or_snippet_scan(): diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 4725bbff..3a4fb034 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -101,7 +101,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, - ca_cert: str = None, pac: PACFile = None, retry: int = 5 + ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning @@ -117,10 +117,11 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.hidden_files_folders = hidden_files_folders self.scan_options = scan_options self._skip_snippets = True if not scan_options & ScanType.SCAN_SNIPPETS.value else False + self.hpsm = hpsm ver_details = Scanner.version_details() self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, - all_extensions=all_extensions, obfuscate=obfuscate + all_extensions=all_extensions, obfuscate=obfuscate, hpsm=self.hpsm ) self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 708a5a55..b12257dc 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -60,6 +60,10 @@ ".pdf", ".min.js", ".mf", ".sum" } +CRC8_MAXIM_DOW_TABLE_SIZE = 0x100 +CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C # 0x31 reflected +CRC8_MAXIM_DOW_INITIAL = 0x00 # 0x00 reflected +CRC8_MAXIM_DOW_FINAL = 0x00 # 0x00 reflected class Winnowing(ScanossBase): """ @@ -105,7 +109,8 @@ class Winnowing(ScanossBase): """ def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = False, quiet: bool = False, - skip_snippets: bool = False, post_size: int = 64, all_extensions: bool = False, obfuscate: bool = False + skip_snippets: bool = False, post_size: int = 64, all_extensions: bool = False, + obfuscate: bool = False, hpsm: bool = False ): """ Instantiate Winnowing class @@ -122,6 +127,9 @@ def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = F self.obfuscate = obfuscate self.ob_count = 1 self.file_map = {} if obfuscate else None + self.hpsm = hpsm + if hpsm: + self.crc8_maxim_dow_table = [] @staticmethod def __normalize(byte): @@ -236,6 +244,10 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')): return wfp + # Add HPSM + if self.hpsm: + hpsm = self.calc_hpsm(contents) + wfp += 'hpsm={0}\n'.format(hpsm) # Initialize variables gram = '' window = [] @@ -297,6 +309,56 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: self.print_stderr(f'Warning: No WFP content data for {file}') return wfp + def calc_hpsm(self, content): + list_normalized = [] #Array of numbers + crc_lines = [] #Array of numbers that represent the crc8_maxim for each line of the file + last_line = 0 + self.crc8_generate_table() + for i, byte in enumerate(content): + c = byte + if c == ASCII_LF: #When there is a new line + if len(list_normalized): + crc_lines.append(self.crc8_buffer(list_normalized)) + list_normalized=[] + elif last_line+1 == i: + crc_lines.append(0xFF) + elif i-last_line > 1: + crc_lines.append(0x00) + last_line = i + else: + c_normalized = self.__normalize(c) + if c_normalized != 0: + list_normalized.append(c_normalized) + crc_lines_hex = [] + for x in crc_lines: + crc_lines_hex.append(hex(x)) + hpsm = ''.join('{:02x}'.format(x) for x in crc_lines) + return hpsm + + def crc8_generate_table(self): + for i in range(CRC8_MAXIM_DOW_TABLE_SIZE): + self.crc8_maxim_dow_table.append(self.crc8_byte_checksum(0, i)) + + def crc8_byte_checksum(self, crc, byte): + crc ^= byte + for count in range(8): + isSet = crc & 0x01 + crc >>= 1 + if isSet: + crc ^= CRC8_MAXIM_DOW_POLYNOMIAL + return crc + + def crc8_byte(self, crc, byte): + index = byte ^ crc + return self.crc8_maxim_dow_table[ index ] ^ ( crc >> 8 ) + + def crc8_buffer(self, buffer): + crc = CRC8_MAXIM_DOW_INITIAL + for index in range(len(buffer)): + crc = self.crc8_byte(crc, buffer[index]) + crc ^= CRC8_MAXIM_DOW_FINAL + return crc + # # End of Winnowing Class # From f9d852366ec05a73b253d6b527083f39998a7d11 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 6 Jul 2023 14:58:12 +0100 Subject: [PATCH 113/489] add min req for hpsm --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 92e4f0fb..c1651c81 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,7 +36,7 @@ install_requires = [options.extras_require] fast_winnowing = - scanoss_winnowing + scanoss_winnowing>=0.2.0 [options.packages.find] where = src From bca9487666b8b9db42997abe14dfc226ddd76c54 Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 6 Jul 2023 14:58:30 +0100 Subject: [PATCH 114/489] fixed issue with CSV generation --- CHANGELOG.md | 7 +++++++ src/scanoss/__init__.py | 2 +- src/scanoss/csvoutput.py | 37 +++++++++++++++++++++++++++---------- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e470d37b..f4685e58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.6.1] - 2023-07-06 +### Fixed +- Fixed issue with CSV dependency generation +- Increased `scanoss-winnowing` minimum requirement to match HPSM support + ## [1.6.0] - 2023-06-16 ### Added - Added support for High Precision Snippet Matching (`--hpsm` or `-H`) while scanning @@ -242,3 +247,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.5.0]: https://github.com/scanoss/scanoss.py/compare/v1.4.2...v1.5.0 [1.5.1]: https://github.com/scanoss/scanoss.py/compare/v1.5.0...v1.5.1 [1.5.2]: https://github.com/scanoss/scanoss.py/compare/v1.5.1...v1.5.2 +[1.6.0]: https://github.com/scanoss/scanoss.py/compare/v1.5.2...v1.6.0 +[1.6.1]: https://github.com/scanoss/scanoss.py/compare/v1.6.0...v1.6.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 3bbcb66e..0ea58347 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.6.0' +__version__ = '1.6.1' diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index 2f66d643..cda12d1a 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -72,6 +72,7 @@ def parse(self, data: json): self.print_stderr(f'Warning: No Dependencies found for {f}: {file_details}') continue for deps in dependencies: + detected = {} purl = deps.get("purl") if not purl: self.print_stderr(f'Warning: No PURL found for {f}: {deps}') @@ -90,6 +91,19 @@ def parse(self, data: json): detected['licenses'] = '' else: detected['licenses'] = ';'.join(dc) + # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl + csv_dict.append({'inventory_id': row_id, 'path': f, 'detected_usage': id_details, + 'detected_component': detected.get('component'), + 'detected_license': detected.get('licenses'), + 'detected_version': detected.get('version'), + 'detected_latest': detected.get('latest'), + 'detected_purls': detected.get('purls'), + 'detected_url': detected.get('url'), + 'detected_path': detected.get('file', ''), + 'detected_match': matched, 'detected_lines': lines, + 'detected_oss_lines': oss_lines + }) + row_id = row_id + 1 else: purls = d.get('purl') if not purls: @@ -116,16 +130,19 @@ def parse(self, data: json): detected['licenses'] = '' else: detected['licenses'] = ';'.join(dc) - # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl - csv_dict.append({'inventory_id': row_id, 'path': f, 'detected_usage': id_details, - 'detected_component': detected.get('component'), - 'detected_license': detected.get('licenses'), - 'detected_version': detected.get('version'), 'detected_latest': detected.get('latest'), - 'detected_purls': detected.get('purls'), 'detected_url': detected.get('url'), - 'detected_path': detected.get('file', ''), - 'detected_match': matched, 'detected_lines': lines, 'detected_oss_lines': oss_lines - }) - row_id = row_id + 1 + # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl + csv_dict.append({'inventory_id': row_id, 'path': f, 'detected_usage': id_details, + 'detected_component': detected.get('component'), + 'detected_license': detected.get('licenses'), + 'detected_version': detected.get('version'), + 'detected_latest': detected.get('latest'), + 'detected_purls': detected.get('purls'), + 'detected_url': detected.get('url'), + 'detected_path': detected.get('file', ''), + 'detected_match': matched, 'detected_lines': lines, + 'detected_oss_lines': oss_lines + }) + row_id = row_id + 1 return csv_dict def produce_from_file(self, json_file: str, output_file: str = None) -> bool: From 723a008d4c986965f1e8c6fa14ecdf738f4844de Mon Sep 17 00:00:00 2001 From: Sean Egan Date: Thu, 27 Jul 2023 09:36:46 +0100 Subject: [PATCH 115/489] added installation instructions for pipx --- CLIENT_HELP.md | 16 ++++++++++++++++ PACKAGE.md | 15 +++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 58390091..89c972d6 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -1,6 +1,22 @@ # SCANOSS Client Usage Help This file contains useful tips/tricks for getting the most out of the SCANOSS platform using the Python client/SDK. +## Installation +### Externally Managed Environments Error +If installing on Ubuntu 2023.04, Fedora 38, Debian 11, etc. a few additional steps are required before installing `scanoss-py`. More details can be found [here](https://itsfoss.com/externally-managed-environment/). + +The recommended method is to install `pipx` and use it to install `scanoss-py`: +```bash +sudo apt install pipx +pipx ensurepath +``` + +This will install the `pipx` package manager, which can then be used to install `scanoss-py`: +```bash +pipx install scanoss[fast_winnowing] +``` +This will install the `scanoss-py` app in a separate virtual environment and create a link to the local path for execution. + ## Certificate Management The SCANOSS SaaS platform runs over HTTPS with publicly signed SSL certificates. However, on-premise installations, or those with a proxy in the middle might be leveraging self-signed versions. diff --git a/PACKAGE.md b/PACKAGE.md index 84d46f40..b3f0e083 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -25,6 +25,21 @@ pip3 install scanoss[fast_winnowing] Alternatively, there is a docker image of the compiled package. It can be found [here](https://github.com/scanoss/scanoss.py/pkgs/container/scanoss-py). Details of how to run it can be found [here](https://github.com/scanoss/scanoss.py/blob/main/GHCR.md). +### Externally Managed Environments on Linux +If installing on Ubuntu 2023.04, Fedora 38, Debian 11, etc. a few additional steps are required before installing `scanoss-py`. More details can be found [here](https://itsfoss.com/externally-managed-environment/). + +The recommended method is to install `pipx` and use it to install `scanoss-py`: +```bash +sudo apt install pipx +pipx ensurepath +``` + +This will install the `pipx` package manager, which can then be used to install `scanoss-py`: +```bash +pipx install scanoss[fast_winnowing] +``` +This will install the `scanoss-py` app in a separate virtual environment and create a link to the local path for execution. + ## Usage The package can be run from the command line, or consumed from another Python script. From 8e301c5a01f48ce21db008331893dce8257b4dc7 Mon Sep 17 00:00:00 2001 From: Marc-Etienne Vargenau Date: Thu, 27 Jul 2023 12:18:39 +0200 Subject: [PATCH 116/489] Update message to version 1.6.1 Signed-off-by: Marc-Etienne Vargenau --- PACKAGE.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/PACKAGE.md b/PACKAGE.md index b3f0e083..9848cf17 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -57,24 +57,32 @@ Running the bare command will list the available sub-commands: ```bash > scanoss-py -usage: scanoss-py [-h] {version,ver,scan,sc,fingerprint,fp,wfp,dependencies,dp,dep} ... +usage: scanoss-py [-h] [--version] + {version,ver,scan,sc,fingerprint,fp,wfp,dependencies,dp,dep,file_count,fc,convert,cv,cnv,cvrt,component,comp,utils,ut} + ... -SCANOSS Python CLI. Ver: 0.9.0, License: MIT +SCANOSS Python CLI. Ver: 1.6.1, License: MIT, Fast Winnowing: True -optional arguments: +options: -h, --help show this help message and exit + --version, -v Display version details Sub Commands: valid subcommands - {version,ver,scan,sc,fingerprint,fp,wfp,dependencies,dp,dep} + {version,ver,scan,sc,fingerprint,fp,wfp,dependencies,dp,dep,file_count,fc,convert,cv,cnv,cvrt,component,comp,utils,ut} sub-command help version (ver) SCANOSS version scan (sc) Scan source code fingerprint (fp, wfp) Fingerprint source code dependencies (dp, dep) - Scan source code for dependencies + Scan source code for dependencies, but do not decorate them + file_count (fc) Search the source tree and produce a file type summary + convert (cv, cnv, cvrt) + Convert file format + component (comp) Component support commands + utils (ut) General utility support commands ``` From there it is possible to scan a source code folder: From f605cc9c0a2b289421efe55c25d15eaf2a2e5abf Mon Sep 17 00:00:00 2001 From: eeisegn Date: Fri, 11 Aug 2023 15:53:07 +0100 Subject: [PATCH 117/489] added .woff2 to skip list --- .github/workflows/container-publish-ghcr.yml | 1 + CHANGELOG.md | 5 +++++ Dockerfile | 2 +- src/scanoss/__init__.py | 2 +- src/scanoss/scanner.py | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 94d5fccf..10a6bb12 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -74,6 +74,7 @@ jobs: platforms: linux/amd64,linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max + provenance: false # Test the docker image - name: Test Published Image diff --git a/CHANGELOG.md b/CHANGELOG.md index f4685e58..07396e8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.6.2] - 2023-08-11 +### Added +- Added `.woff2` to the list of file type to skip while scanning + ## [1.6.1] - 2023-07-06 ### Fixed - Fixed issue with CSV dependency generation @@ -249,3 +253,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.5.2]: https://github.com/scanoss/scanoss.py/compare/v1.5.1...v1.5.2 [1.6.0]: https://github.com/scanoss/scanoss.py/compare/v1.5.2...v1.6.0 [1.6.1]: https://github.com/scanoss/scanoss.py/compare/v1.6.0...v1.6.1 +[1.6.2]: https://github.com/scanoss/scanoss.py/compare/v1.6.1...v1.6.2 diff --git a/Dockerfile b/Dockerfile index f521aa60..95dc4617 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-buster as base +FROM --platform=$BUILDPLATFORM python:3.10-slim-buster as base LABEL maintainer="SCANOSS " LABEL org.opencontainers.image.source=https://github.com/scanoss/scanoss.py diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 0ea58347..b0ebcc9b 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.6.1' +__version__ = '1.6.2' diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 3a4fb034..c69e0614 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -72,7 +72,7 @@ ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", - ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".xht", + ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".woff2", ".xht", ".xhtml", ".xls", ".xlsx", ".xml", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", From 7743dda7ccfffef38c90096f7af25ec7c3ac2883 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Tue, 22 Aug 2023 19:47:54 +0200 Subject: [PATCH 118/489] Scan perf improvements (#25) * change default post size and disable size limiting * reduced default thread count, post size, and added file limit to threaded scanning * increase fast winnowing requirement --- setup.cfg | 2 +- src/scanoss/cli.py | 16 ++++---- src/scanoss/scanner.py | 32 ++++++++++----- src/scanoss/scanossapi.py | 4 +- src/scanoss/winnowing.py | 83 +++++++++++++++++++++++++++------------ 5 files changed, 90 insertions(+), 47 deletions(-) diff --git a/setup.cfg b/setup.cfg index c1651c81..e4c40f75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,7 +36,7 @@ install_requires = [options.extras_require] fast_winnowing = - scanoss_winnowing>=0.2.0 + scanoss_winnowing>=0.3.0 [options.packages.find] where = src diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index bc9f0a82..ae1c809f 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -77,8 +77,8 @@ def setup_args() -> None: p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_scan.add_argument('--format', '-f', type=str, choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], help='Result output format (optional - default: plain)') - p_scan.add_argument('--threads', '-T', type=int, default=10, - help='Number of threads to use while scanning (optional - default 10)') + p_scan.add_argument('--threads', '-T', type=int, default=5, + help='Number of threads to use while scanning (optional - default 5)') p_scan.add_argument('--flags', '-F', type=int, help='Scanning engine flags (1: disable snippet matching, 2 enable snippet ids, ' '4: disable dependencies, 8: disable licenses, 16: disable copyrights,' @@ -87,10 +87,10 @@ def setup_args() -> None: '1024: enable download_url, 2048: enable GitHub full path, ' '4096: disable extended server stats)') p_scan.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') - p_scan.add_argument('--post-size', '-P', type=int, default=64, - help='Number of kilobytes to limit the post to while scanning (optional - default 64)') - p_scan.add_argument('--timeout', '-M', type=int, default=120, - help='Timeout (in seconds) for API communication (optional - default 120)') + p_scan.add_argument('--post-size', '-P', type=int, default=32, + help='Number of kilobytes to limit the post to while scanning (optional - default 32)') + p_scan.add_argument('--timeout', '-M', type=int, default=180, + help='Timeout (in seconds) for API communication (optional - default 180)') p_scan.add_argument('--retry', '-R', type=int, default=5, help='Retry limit for API communication (optional - default 5)') p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') @@ -445,9 +445,9 @@ def scan(parser, args): print_stderr("Scanning all hidden files/folders...") if args.skip_snippets: print_stderr("Skipping snippets...") - if args.post_size != 64: + if args.post_size != 32: print_stderr(f'Changing scanning POST size to: {args.post_size}k...') - if args.timeout != 120: + if args.timeout != 180: print_stderr(f'Changing scanning POST timeout to: {args.timeout}...') if args.retry != 5: print_stderr(f'Changing scanning POST retry to: {args.retry}...') diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index c69e0614..7e30d627 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -97,7 +97,7 @@ class Scanner(ScanossBase): def __init__(self, wfp: str = None, scan_output: str = None, output_format: str = 'plain', debug: bool = False, trace: bool = False, quiet: bool = False, api_key: str = None, url: str = None, sbom_path: str = None, scan_type: str = None, flags: str = None, nb_threads: int = 5, - post_size: int = 64, timeout: int = 120, no_wfp_file: bool = False, + post_size: int = 32, timeout: int = 180, no_wfp_file: bool = False, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, @@ -141,6 +141,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str else: self.threaded_scan = None self.max_post_size = post_size * 1024 if post_size > 0 else MAX_POST_SIZE # Set the max post size (default 64k) + self.post_file_count = post_size if post_size > 0 else 32 # Max number of files for any given POST (default 32) if self._skip_snippets: self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets @@ -360,11 +361,13 @@ def scan_folder(self, scan_dir: str) -> bool: spinner = None if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') + save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan wfp_list = [] scan_block = '' scan_size = 0 queue_size = 0 - file_count = 0 + file_count = 0 # count all files fingerprinted + wfp_file_count = 0 # count number of files in each queue post scan_started = False for root, dirs, files in os.walk(scan_dir): self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') @@ -389,7 +392,9 @@ def scan_folder(self, scan_dir: str) -> bool: wfp = self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) if wfp is None or wfp == '': self.print_stderr(f'Warning: No WFP returned for {path}') - wfp_list.append(wfp) + continue + if save_wfps_for_print: + wfp_list.append(wfp) file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode("utf-8")) @@ -398,13 +403,17 @@ def scan_folder(self, scan_dir: str) -> bool: self.threaded_scan.queue_add(scan_block) queue_size += 1 scan_block = '' + wfp_file_count = 0 scan_block += wfp scan_size = len(scan_block.encode("utf-8")) - if scan_size >= self.max_post_size: + wfp_file_count += 1 + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 scan_block = '' - if queue_size > self.nb_threads and not scan_started: # Start scanning if we have something to do + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( @@ -416,8 +425,8 @@ def scan_folder(self, scan_dir: str) -> bool: if spinner: spinner.finish() - if wfp_list: - if not self.no_wfp_file or not self.threaded_scan: # Write a WFP file if no threading is requested + if file_count > 0: + if save_wfps_for_print: # Write a WFP file if no threading is requested self.print_debug(f'Writing fingerprints to {self.wfp}') with open(self.wfp, 'w') as f: f.write(''.join(wfp_list)) @@ -730,7 +739,8 @@ def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> boo raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") cur_size = 0 queue_size = 0 - file_count = 0 + file_count = 0 # count all files fingerprinted + wfp_file_count = 0 # count number of files in each queue post scan_started = False wfp = '' scan_block = '' @@ -742,17 +752,19 @@ def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> boo cur_size = len(wfp.encode("utf-8")) scan_block = line # Start storing the next file file_count += 1 + wfp_file_count += 1 else: scan_block += line # Store the rest of the WFP for this file l_size = cur_size + len(scan_block.encode('utf-8')) # Hit the max post size, so sending the current batch and continue processing - if l_size >= self.max_post_size and wfp: + if (wfp_file_count > self.post_file_count or l_size >= self.max_post_size) and wfp: if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') self.threaded_scan.queue_add(wfp) queue_size += 1 wfp = '' - if queue_size > self.nb_threads and not scan_started: # Start scanning if we have something to do + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 4d803e8d..9f4248aa 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -52,7 +52,7 @@ class ScanossApi(ScanossBase): def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None, url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - timeout: int = 120, ver_details: str = None, ignore_cert_errors: bool = False, + timeout: int = 180, ver_details: str = None, ignore_cert_errors: bool = False, proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5): """ Initialise the SCANOSS API @@ -81,7 +81,7 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st self.scan_format = scan_format if scan_format else 'plain' self.sbom_path = sbom_path self.flags = flags - self.timeout = timeout if timeout > 5 else 120 + self.timeout = timeout if timeout > 5 else 180 self.retry_limit = retry if retry >= 0 else 5 self.ignore_cert_errors = ignore_cert_errors self.headers = {} diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index b12257dc..47ca74fb 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -57,13 +57,14 @@ ".o", ".a", ".so", ".obj", ".dll", ".lib", ".out", ".app", ".bin", ".lst", ".dat", ".json", ".htm", ".html", ".xml", ".md", ".txt", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".pages", ".key", ".numbers", - ".pdf", ".min.js", ".mf", ".sum" + ".pdf", ".min.js", ".mf", ".sum", ".woff", ".woff2" } CRC8_MAXIM_DOW_TABLE_SIZE = 0x100 -CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C # 0x31 reflected -CRC8_MAXIM_DOW_INITIAL = 0x00 # 0x00 reflected -CRC8_MAXIM_DOW_FINAL = 0x00 # 0x00 reflected +CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C # 0x31 reflected +CRC8_MAXIM_DOW_INITIAL = 0x00 # 0x00 reflected +CRC8_MAXIM_DOW_FINAL = 0x00 # 0x00 reflected + class Winnowing(ScanossBase): """ @@ -108,8 +109,8 @@ class Winnowing(ScanossBase): a list of WFP fingerprints with their corresponding line numbers. """ - def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = False, quiet: bool = False, - skip_snippets: bool = False, post_size: int = 64, all_extensions: bool = False, + def __init__(self, size_limit: bool = False, debug: bool = False, trace: bool = False, quiet: bool = False, + skip_snippets: bool = False, post_size: int = 32, all_extensions: bool = False, obfuscate: bool = False, hpsm: bool = False ): """ @@ -117,7 +118,7 @@ def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = F Parameters ---------- size_limit: bool - Limit the size of a fingerprint to 64k (post size) - Default True + Limit the size of a fingerprint to 32k (post size) - Default False """ super().__init__(debug, trace, quiet) self.size_limit = size_limit @@ -130,6 +131,7 @@ def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = F self.hpsm = hpsm if hpsm: self.crc8_maxim_dow_table = [] + self.crc8_generate_table() @staticmethod def __normalize(byte): @@ -285,7 +287,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: if self.size_limit and \ (len(wfp.encode("utf-8")) + len( output.encode("utf-8"))) > self.max_post_size: - self.print_debug(f'Truncating WFP (64k limit) for: {file}') + self.print_debug(f'Truncating WFP ({self.max_post_size} limit) for: {file}') output = '' break # Stop collecting snippets as it's over 64k wfp += output + '\n' @@ -310,53 +312,82 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: return wfp def calc_hpsm(self, content): - list_normalized = [] #Array of numbers - crc_lines = [] #Array of numbers that represent the crc8_maxim for each line of the file + """ + Calculate the HPSM data for this content + + :param content: content bytes to calculate + :return: HPSM encoded data + """ + list_normalized = [] # Array of numbers + crc_lines = [] # Array of numbers that represent the crc8_maxim for each line of the file last_line = 0 - self.crc8_generate_table() for i, byte in enumerate(content): c = byte - if c == ASCII_LF: #When there is a new line + if c == ASCII_LF: # When there is a new line if len(list_normalized): crc_lines.append(self.crc8_buffer(list_normalized)) - list_normalized=[] + list_normalized = [] elif last_line+1 == i: crc_lines.append(0xFF) - elif i-last_line > 1: + elif i-last_line > 1: crc_lines.append(0x00) last_line = i else: c_normalized = self.__normalize(c) if c_normalized != 0: list_normalized.append(c_normalized) - crc_lines_hex = [] - for x in crc_lines: - crc_lines_hex.append(hex(x)) hpsm = ''.join('{:02x}'.format(x) for x in crc_lines) return hpsm def crc8_generate_table(self): - for i in range(CRC8_MAXIM_DOW_TABLE_SIZE): - self.crc8_maxim_dow_table.append(self.crc8_byte_checksum(0, i)) - - def crc8_byte_checksum(self, crc, byte): + """ + Generate the CRC8 maxim dow table + + :return: nothing + """ + if not self.crc8_maxim_dow_table or len(self.crc8_maxim_dow_table) == 0: + for i in range(CRC8_MAXIM_DOW_TABLE_SIZE): + self.crc8_maxim_dow_table.append(self.crc8_byte_checksum(0, i)) + + @staticmethod + def crc8_byte_checksum(crc: int, byte): + """ + Calculate the CRC8 checksum for the given byte + + :param crc: + :param byte: + :return: CRC for the byte + """ crc ^= byte for count in range(8): - isSet = crc & 0x01 + is_set = crc & 0x01 crc >>= 1 - if isSet: + if is_set: crc ^= CRC8_MAXIM_DOW_POLYNOMIAL return crc - def crc8_byte(self, crc, byte): + def crc8_byte(self, crc: int, byte): + """ + Calculate the CRC8 for the given CRC & Byte + + :param crc: + :param byte: + :return: + """ index = byte ^ crc - return self.crc8_maxim_dow_table[ index ] ^ ( crc >> 8 ) + return self.crc8_maxim_dow_table[index] ^ (crc >> 8) def crc8_buffer(self, buffer): + """ + Calculate the CRC for the given buffer list + + :param buffer: + :return: + """ crc = CRC8_MAXIM_DOW_INITIAL for index in range(len(buffer)): crc = self.crc8_byte(crc, buffer[index]) - crc ^= CRC8_MAXIM_DOW_FINAL + crc ^= CRC8_MAXIM_DOW_FINAL # Bitwise OR (XOR) of crc in Maxim Dow Final return crc # From 2a773c6f04602c5870e07a03517c48a2426c75e7 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Tue, 22 Aug 2023 19:10:56 +0100 Subject: [PATCH 119/489] v1.6.2 release --- CHANGELOG.md | 7 +++++++ src/scanoss/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07396e8f..be22695a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.6.3] - 2023-08-22 +### Changed +- Changed default scan POST size to 32k +- Changed default scanning threads to 5 (and timeout to 180 seconds) +- Improved HPSM generation performance + ## [1.6.2] - 2023-08-11 ### Added - Added `.woff2` to the list of file type to skip while scanning @@ -254,3 +260,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.6.0]: https://github.com/scanoss/scanoss.py/compare/v1.5.2...v1.6.0 [1.6.1]: https://github.com/scanoss/scanoss.py/compare/v1.6.0...v1.6.1 [1.6.2]: https://github.com/scanoss/scanoss.py/compare/v1.6.1...v1.6.2 +[1.6.3]: https://github.com/scanoss/scanoss.py/compare/v1.6.2...v1.6.3 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b0ebcc9b..8d8e29ae 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.6.2' +__version__ = '1.6.3' From e7c874fa44bb0367494d0b36322dba442eac2551 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Tue, 22 Aug 2023 19:10:56 +0100 Subject: [PATCH 120/489] v1.6.3 release --- .github/dependabot.yml | 6 ++++++ CHANGELOG.md | 7 +++++++ src/scanoss/__init__.py | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..1c601a3a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: monthly diff --git a/CHANGELOG.md b/CHANGELOG.md index 07396e8f..be22695a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.6.3] - 2023-08-22 +### Changed +- Changed default scan POST size to 32k +- Changed default scanning threads to 5 (and timeout to 180 seconds) +- Improved HPSM generation performance + ## [1.6.2] - 2023-08-11 ### Added - Added `.woff2` to the list of file type to skip while scanning @@ -254,3 +260,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.6.0]: https://github.com/scanoss/scanoss.py/compare/v1.5.2...v1.6.0 [1.6.1]: https://github.com/scanoss/scanoss.py/compare/v1.6.0...v1.6.1 [1.6.2]: https://github.com/scanoss/scanoss.py/compare/v1.6.1...v1.6.2 +[1.6.3]: https://github.com/scanoss/scanoss.py/compare/v1.6.2...v1.6.3 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b0ebcc9b..8d8e29ae 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.6.2' +__version__ = '1.6.3' From 3129e053bdca5119eecc45da9be94c61c51e1ae3 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Fri, 15 Sep 2023 15:12:28 +0200 Subject: [PATCH 121/489] Comp commands (#26) * added comp vulns command * added comp search and versions command * setting default URL config * added testing/sample files * fix sub-command help * updating comp help docs --- CHANGELOG.md | 8 ++ CLIENT_HELP.md | 107 ++++++++++++++ README.md | 13 ++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 151 +++++++++++++++++--- src/scanoss/components.py | 128 +++++++++++++++++ src/scanoss/scanner.py | 5 +- src/scanoss/scanossgrpc.py | 109 +++++++++++++- tests/data/comp-search-input.json | 5 + tests/data/comp-versions-input.json | 4 + tests/data/package.json | 21 +++ tests/data/purl-input-with-requirement.json | 8 ++ tests/data/purl-input.json | 7 + 13 files changed, 538 insertions(+), 30 deletions(-) create mode 100644 tests/data/comp-search-input.json create mode 100644 tests/data/comp-versions-input.json create mode 100644 tests/data/package.json create mode 100644 tests/data/purl-input-with-requirement.json create mode 100644 tests/data/purl-input.json diff --git a/CHANGELOG.md b/CHANGELOG.md index be22695a..25ce0fbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.7.0] - 2023-09-15 +### Added +- Added Component Decoration sub-commands: + - Search (`scanoss-py comp search`) + - Versions (`scanoss-py comp versions`) + - Vulnerabilities (`scanoss-py comp vulns`) + ## [1.6.3] - 2023-08-22 ### Changed - Changed default scan POST size to 32k @@ -261,3 +268,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.6.1]: https://github.com/scanoss/scanoss.py/compare/v1.6.0...v1.6.1 [1.6.2]: https://github.com/scanoss/scanoss.py/compare/v1.6.1...v1.6.2 [1.6.3]: https://github.com/scanoss/scanoss.py/compare/v1.6.2...v1.6.3 +[1.7.0]: https://github.com/scanoss/scanoss.py/compare/v1.6.3...v1.7.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 89c972d6..2d42a972 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -129,3 +129,110 @@ pip3 install --upgrade https://github.com/pietrodn/grpcio-mac-arm-build/releases ``` This command above will install `grpcio` `1.5.1` for Python `3.9`. To install for `3.10` simply replace the `cp39` with `cp310`. + +## Command Execution +There are multiple commands (and sub commands) available through `scanoss-py`. +Detailed help is available for all directly from the CLI itself: +```bash +scanoss-py --help +scanoss-py scan --help +scanoss-py comp +scanoss-py comp vulns --help +scanoss-py utils +``` + +### Fingerprint a project folder +The following command provides the capability to fingerprint (generate WFPs) for a given file/folder: +```bash +scanoss-py wfp --help +``` +The following command fingerprints the `src` folder and writes the output to `src-fingers.wfp`: +```bash +scanoss-py wfp -o src-fingers.wfp src +``` + +This fingerprint (WFP) can then be sent to the SCANOSS engine using the scanning command: +```bash +scanoss-py scan -w src-fingers.wfp -o scan-results.json +``` + +### Scan a project folder +The following command provides the capability to scan a given file/folder: +```bash +scanoss-py scan --help +``` + +The following command scans the `src` folder and writes the output to `scan-results.json`: +```bash +scanoss-py scan -o scan-results.json src +``` + +### Converting RAW results into other formats +The following command provides the capability to convert the RAW scan results from a SCANOSS scan into multiple different formats, including CycloneDX, SPDX Lite, CSV, etc. +For the full set of formats, please run: +```bash +scanoss-py cnv --help +``` + +The following command converts `scan-results.json` to SPDX Lite: +```bash +scanoss-py cnv --input scan-results.json --format spdxlite --output scan-results-spdxlite.json +``` + +### Component Commands +The `component` command has a suite of sub-commands designed to operate on OSS components. For example: +* Vulnerabilities (`vulns`) +* Search (`search`) +* Version Details (`versions`) +* Cryptography (`crypto`) + +For the latest list of sub-commands, please run: +```bash +scanoss-py comp --help +``` + +#### Component Vulnerabilities +The following command provides the capability to search the SCANOSS KB for component vulnerabilities: +```bash +scanoss-py comp vulns -p "pkg:github/unoconv/unoconv" +``` +It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): +```bash +scanoss-py comp vulns -i purl-input.json -o vulnernable-comps.json +``` + + +#### Component Search +The following command provides the capability to search the SCANOSS KB for an Open Source component: +```bash +scanoss-py comp search -s "unoconv" +``` +This command will search through different combinations to retrieve a proposed list of components (i.e. vendor/component, component, vendor, purl). + +It is also possible to search by component and vendor, while restricting the package type: +```bash +scanoss-py comp search --key $SC_API_KEY -c unoconv -v unoconv -p github +``` + +**Note:** This sub-command requires a subscription to SCANOSS premium data. + +#### Component Versions +The following command provides the capability to search the SCANOSS KB for versions of a specified component PURL: +```bash +scanoss-py comp versions --key $SC_API_KEY -p "pkg:github/unoconv/unoconv" +``` + +**Note:** This sub-command requires a subscription to SCANOSS premium data. + +#### Cryptographic Algorithms +The following command provides the capability to search the SCANOSS KB for any cryptographic algorithms detected in a specified component PURL: +```bash +scanoss-py comp crypto --key $SC_API_KEY -p "pkg:github/unoconv/unoconv" +``` +It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): +```bash +scanoss-py comp crypto --key $SC_API_KEY -i purl-input.json -o crypto-components.json +``` + +**Note:** This sub-command requires a subscription to SCANOSS premium data. + diff --git a/README.md b/README.md index 52f2e8a3..4d164312 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,19 @@ The package will then be available to install using: pip3 install scanoss ``` +##### GitHub Actions +There are a number of [workflows](.github/workflows) setup for this repository. They provide the following: +* [Local build/test](.github/workflows/python-local-test.yml) + * Automatically triggered on pushes or PRs to main. Can also be run manually for other branches +* [Local container build/test](.github/workflows/container-local-test.yml) + * Automatically triggered on pushes or PRs to main. Can also be run manually for other branches +* [Publish to Test PyPI](.github/workflows/python-publish-testpypi.yml) + * Can be manually triggered to push a test version from any branch +* [Publish to PyPI](.github/workflows/python-publish-pypi.yml) + * Build and publish the Python package to PyPI (triggered by v*.*.* tag) +* [Publish container to GHCR](.github/workflows/container-publish-ghcr.yml) + * Build and publish the Python container to GHCR (triggered by v*.*.* tag) + ## Bugs/Features To request features or alert about bugs, please do so [here](https://github.com/scanoss/scanoss.py/issues). diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 8d8e29ae..3ede3655 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.6.3' +__version__ = '1.7.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ae1c809f..362694a9 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -114,7 +114,7 @@ def setup_args() -> None: p_wfp.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_wfp.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, - help='Fingerprint the file contents supplied via STDIN (optional)') + help='Fingerprint the file contents supplied via STDIN (optional)') p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') @@ -161,32 +161,61 @@ def setup_args() -> None: description=f'SCANOSS Component commands: {__version__}', help='Component support commands') - comp_sub = p_comp.add_subparsers(title='Component Commands', dest='subparsercmd', description='utils sub-commands', + comp_sub = p_comp.add_subparsers(title='Component Commands', dest='subparsercmd', description='component sub-commands', help='component sub-commands') # Component Sub-command: component crypto c_crypto = comp_sub.add_parser('crypto', aliases=['cr'], description=f'Show Cryptographic algorithms: {__version__}', - help='Retreive the cryptographic algorithms for the given components') + help='Retreive cryptographic algorithms for the given components') c_crypto.set_defaults(func=comp_crypto) - c_crypto.add_argument('--purl', '-p', type=str, nargs="*", help='Package URL - PURL to process.') - c_crypto.add_argument('--input', '-i', type=str, help='Input file name') - c_crypto.add_argument('--output','-o', type=str, help='Output result file name (optional - default stdout).') - c_crypto.add_argument('--timeout', '-M', type=int, default=600, - help='Timeout (in seconds) for API communication (optional - default 600)') + c_vulns = comp_sub.add_parser('vulns', aliases=['vulnerabilities', 'vu'], + description=f'Show Vulnerability details: {__version__}', + help='Retreive vulnerabilities for the given components') + c_vulns.set_defaults(func=comp_vulns) + + c_search = comp_sub.add_parser('search', aliases=['sc'], + description=f'Search component details: {__version__}', + help='Search for a KB component') + c_search.add_argument('--input', '-i', type=str, help='Input file name') + c_search.add_argument('--search', '-s', type=str, help='Generic component search') + c_search.add_argument('--vendor', '-v', type=str, help='Generic component search') + c_search.add_argument('--comp', '-c', type=str, help='Generic component search') + c_search.add_argument('--package', '-p', type=str, help='Generic component search') + c_search.add_argument('--limit', '-l', type=int, help='Generic component search') + c_search.add_argument('--offset', '-f', type=int, help='Generic component search') + c_search.set_defaults(func=comp_search) + + c_versions = comp_sub.add_parser('versions', aliases=['vs'], + description=f'Get component version details: {__version__}', + help='Search for component versions') + c_versions.add_argument('--input', '-i', type=str, help='Input file name') + c_versions.add_argument('--purl', '-p', type=str, help='Generic component search') + c_versions.add_argument('--limit', '-l', type=int, help='Generic component search') + c_versions.set_defaults(func=comp_versions) + + # Common purl Component sub-command options + for p in [c_crypto, c_vulns]: + p.add_argument('--purl', '-p', type=str, nargs="*", help='Package URL - PURL to process.') + p.add_argument('--input', '-i', type=str, help='Input file name') + # Common Component sub-command options + for p in [c_crypto, c_vulns, c_search, c_versions]: + p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + p.add_argument('--timeout', '-M', type=int, default=600, + help='Timeout (in seconds) for API communication (optional - default 600)') # Sub-command: utils p_util = subparsers.add_parser('utils', aliases=['ut'], description=f'SCANOSS Utility commands: {__version__}', help='General utility support commands') - utils_sub = p_util.add_subparsers(title='Utils Commands', dest='subparsercmd', description='component sub-commands', - help='component sub-commands') + utils_sub = p_util.add_subparsers(title='Utils Commands', dest='subparsercmd', description='utils sub-commands', + help='utils sub-commands') # Utils Sub-command: utils fast p_f_f = utils_sub.add_parser('fast', - description=f'Is fast winnowing enabled: {__version__}', help='SCANOSS fast winnowing') + description=f'Is fast winnowing enabled: {__version__}', help='SCANOSS fast winnowing') p_f_f.set_defaults(func=fast) # Utils Sub-command: utils certloc @@ -223,7 +252,7 @@ def setup_args() -> None: p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') # Global Scan/GRPC options - for p in [p_scan, c_crypto]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions]: p.add_argument('--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)') p.add_argument('--proxy', type=str, help='Proxy URL to use for connections (optional). ' @@ -237,14 +266,14 @@ def setup_args() -> None: '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC') # Global GRPC options - for p in [p_scan, c_crypto]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions]: p.add_argument('--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)') p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' 'Can also use the environment variable "grcp_proxy=:"') # Help/Trace command options - for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto]: + for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto, c_vulns, c_search, c_versions]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -264,6 +293,7 @@ def setup_args() -> None: exit(1) args.func(parser, args) # Execute the function associated with the sub-command + def ver(*_): """ Run the "ver" sub-command @@ -271,6 +301,7 @@ def ver(*_): """ print(f'Version: {__version__}') + def fast(*_): """ Run the "fast" sub-command @@ -278,6 +309,7 @@ def fast(*_): """ print(f'Fast Winnowing: {FAST_WINNOWING}') + def file_count(parser, args): """ Run the "file_count" sub-command @@ -606,14 +638,11 @@ def utils_cert_download(_, args): import traceback file = sys.stdout - hostname = 'unset' - port = 'unkown' if args.output: file = open(args.output, 'w') parsed_url = urlparse(args.hostname) hostname = parsed_url.hostname or args.hostname # Use the parse hostname, or it None use the supplied one port = int(parsed_url.port or args.port) # Use the parsed port, if not use the supplied one (default 443) - certs = [] try: if args.debug: print_stderr(f'Connecting to {hostname} on {port}...') @@ -623,13 +652,13 @@ def utils_cert_download(_, args): certs = conn.get_peer_cert_chain() for index, cert in enumerate(certs): cert_components = dict(cert.get_subject().get_components()) - if(sys.version_info[0] >= 3): + if sys.version_info[0] >= 3: cn = cert_components.get(b'CN') else: cn = cert_components.get('CN') if not args.quiet: print_stderr(f'Centificate {index} - CN: {cn}') - if(sys.version_info[0] >= 3): + if sys.version_info[0] >= 3: print((crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file) # Print the downloaded PEM certificate else: print((crypto.dump_certificate(crypto.FILETYPE_PEM, cert)).strip(), file=file) @@ -708,10 +737,92 @@ def comp_crypto(parser, args): exit(1) pac_file = get_pac_file(args.pac) comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout) if not comps.get_crypto_details(args.input, args.purl, args.output): exit(1) + + +def comp_vulns(parser, args): + """ + Run the "component vulns" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + exit(1) + pac_file = get_pac_file(args.pac) + comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + timeout=args.timeout) + if not comps.get_vulnerabilities(args.input, args.purl, args.output): + exit(1) + + +def comp_search(parser, args): + """ + Run the "component search" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if ((not args.input and not args.search and not args.vendor and not args.comp) or + (args.input and (args.search or args.vendor or args.comp))): + print_stderr('Please specify an input file or search terms (--input or --search, or --vendor or --comp.)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + exit(1) + pac_file = get_pac_file(args.pac) + comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + timeout=args.timeout) + if not comps.search_components(args.output, json_file=args.input, + search=args.search, vendor=args.vendor, comp=args.comp, package=args.package, + limit=args.limit, offset=args.offset): + exit(1) + + +def comp_versions(parser, args): + """ + Run the "component versions" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.input and not args.purl) or (args.input and args.purl): + print_stderr('Please specify an input file or search terms (--input or --purl.)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + exit(1) + pac_file = get_pac_file(args.pac) + comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + timeout=args.timeout) + if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit): + exit(1) + + def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 6efe338f..c7993c77 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -95,6 +95,24 @@ def load_purls(self, json_file: str = None, purls: [] = None) -> dict: return None return purl_request + def load_json(self, json_file: str = None) -> dict: + """ + Load the specified json and return a dictionary + + :param json_file: JSON PURL file + :return: PURL Request dictionary + """ + if json_file: + if not os.path.isfile(json_file) or not os.access(json_file, os.R_OK): + self.print_stderr(f'ERROR: JSON file does not exist, is not a file, or is not readable: {json_file}') + return None + with open(json_file, 'r') as f: + try: + return json.loads(f.read()) + except Exception as e: + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + return None + def _open_file_or_sdtout(self, filename): """ Open the given filename if requested, otherwise return STDOUT @@ -148,3 +166,113 @@ def get_crypto_details(self, json_file: str = None, purls: [] = None, output_fil self.print_msg(f'Results written to: {output_file}') self._close_file(output_file, file) return success + + def get_vulnerabilities(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: + """ + Retrieve any vulnerabilities related to the given PURLs + + :param json_file: PURL JSON request file (optional) + :param purls: PURL request array (optional) + :param output_file: output filename (optional). Default: STDOUT + :return: True on success, False otherwise + """ + success = False + purls_request = self.load_purls(json_file, purls) + if purls_request is None or len(purls_request) == 0: + return False + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + self.print_msg('Sending PURLs to Vulnerability API for decoration...') + response = self.grpc_api.get_vulnerabilities_json(purls_request) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success + + def search_components(self, output_file: str = None, json_file: str = None, + search: str = None, vendor: str = None, comp: str = None, package: str = None, + limit: int = None, offset: int = None) -> bool: + """ + Search for a component based on the given search criteria + + :param output_file: output filename (optional). Default: STDOUT + :param json_file: Search JSON request file (optional) + :param search: Search for (vendor/component/purl) for a component (overrides vendor/component) + :param vendor: Vendor to search for + :param comp: Component to search for + :param package: Package (purl type) to search for. i.e. github/maven/maven/npn/all - default github + :param limit: Number of matches to return + :param offset: Offset to submit to return next (limit) of component matches + :return: True on success, False otherwise + """ + success = False + request: dict + if json_file: # Parse the json file to extract the search details + request = self.load_json(json_file) + if request is None: + return False + else: # Construct a query dictionary from parameters + request = { + "search": search, + "vendor": vendor, + "component": comp, + "package": package + } + if limit is not None and limit > 0: + request["limit"] = limit + if offset is not None and offset > 0: + request["offset"] = offset + + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + self.print_msg('Sending search data to Components API...') + response = self.grpc_api.search_components_json(request) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success + + def get_component_versions(self, output_file: str = None, json_file: str = None, + purl: str = None, limit: int = None) -> bool: + """ + Search for a component versions based on the given search criteria + + :param output_file: output filename (optional). Default: STDOUT + :param json_file: Search JSON request file (optional) + :param purl: PURL to retrieve versions for + :param limit: Number of version to return + :return: True on success, False otherwise + """ + success = False + request: dict + if json_file: # Parse the json file to extract the search details + request = self.load_json(json_file) + if request is None: + return False + else: # Construct a query dictionary from parameters + request = { + "purl": purl + } + if limit is not None and limit > 0: + request["limit"] = limit + + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + self.print_msg('Sending PURLs to Component Versions API...') + response = self.grpc_api.get_component_versions_json(request) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 7e30d627..62c627de 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -41,6 +41,7 @@ from .scanossgrpc import ScanossGrpc from .scantype import ScanType from .scanossbase import ScanossBase +from . import __version__ FAST_WINNOWING = False try: @@ -51,8 +52,6 @@ FAST_WINNOWING = False from .winnowing import Winnowing -from . import __version__ - FILTERED_DIRS = { # Folders to skip "nbproject", "nbbuild", "nbdist", "__pycache__", "venv", "_yardoc", "eggs", "wheels", "htmlcov", "__pypackages__" } @@ -843,7 +842,7 @@ def wfp_contents(self, filename: str, contents: bytes, wfp_file: str = None): else: print(wfp) else: - Scanner.print_stderr(f'Warning: No fingerprints generated for: {scan_file}') + Scanner.print_stderr(f'Warning: No fingerprints generated for: {wfp_file}') def wfp_file(self, scan_file: str, wfp_file: str = None): """ diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 23f1daad..42945248 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -33,16 +33,20 @@ from pypac.resolver import ProxyResolver from urllib.parse import urlparse +from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub +from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, DependencyResponse from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest +from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse +from .api.components.v2.scanoss_components_pb2 import CompSearchRequest, CompSearchResponse, CompVersionRequest, CompVersionResponse from .scanossbase import ScanossBase from . import __version__ -# DEFAULT_URL = "https://osskb.org" -DEFAULT_URL = "https://scanoss.com" +DEFAULT_URL = "https://osskb.org" # default free service URL +DEFAULT_URL2 = "https://scanoss.com" # default premium service URL SCANOSS_GRPC_URL = os.environ.get("SCANOSS_GRPC_URL") if os.environ.get("SCANOSS_GRPC_URL") else DEFAULT_URL SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' @@ -72,9 +76,11 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu """ super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_GRPC_URL + self.api_key = api_key if api_key else SCANOSS_API_KEY + if self.api_key and not url and not os.environ.get("SCANOSS_GRPC_URL"): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.url = self.url.lower() self.orig_url = self.url # Used for proxy lookup - self.api_key = api_key if api_key else SCANOSS_API_KEY self.timeout = timeout self.proxy = proxy self.grpc_proxy = grpc_proxy @@ -99,15 +105,19 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu self.print_debug(f'Setting up (secure: {secure}) connection to {self.url}...') self._get_proxy_config() if secure is False: # insecure connection - self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) + self.comp_search_stub = ComponentsStub(grpc.insecure_channel(self.url)) self.crypto_stub = CryptographyStub(grpc.insecure_channel(self.url)) + self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) + self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url)) else: if ca_cert is not None: credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate else: credentials = grpc.ssl_channel_credentials() # secure connection with default certificate - self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) + self.comp_search_stub = ComponentsStub(grpc.secure_channel(self.url, credentials)) self.crypto_stub = CryptographyStub(grpc.secure_channel(self.url, credentials)) + self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) + self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials)) @classmethod def _load_cert(cls, cert_file: str) -> bytes: @@ -235,7 +245,94 @@ def get_crypto_json(self, purls: dict) -> dict: if resp: if not self._check_status_response(resp.status, request_id): return None - resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dictionary + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict + del resp_dict['status'] + return resp_dict + return None + + def get_vulnerabilities_json(self, purls: dict) -> dict: + """ + Client function to call the rpc for Vulnerability GetVulnerabilities + :param purls: Message to send to the service + :return: Server response or None + """ + if not purls: + self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + return None + request_id = str(uuid.uuid4()) + resp: VulnerabilityResponse + try: + request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...') + resp = self.vuln_stub.GetVulnerabilities(request, metadata=metadata, timeout=self.timeout) + except Exception as e: + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') + else: + if resp: + if not self._check_status_response(resp.status, request_id): + return None + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict + del resp_dict['status'] + return resp_dict + return None + + def search_components_json(self, search: dict) -> dict: + """ + Client function to call the rpc for Components SearchComponents + :param search: Message to send to the service + :return: Server response or None + """ + if not search: + self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + return None + request_id = str(uuid.uuid4()) + resp: CompSearchResponse + try: + request = ParseDict(search, CompSearchRequest()) # Parse the JSON/Dict into the purl request object + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + self.print_debug(f'Sending component search data (rqId: {request_id})...') + resp = self.comp_search_stub.SearchComponents(request, metadata=metadata, timeout=self.timeout) + except Exception as e: + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') + else: + if resp: + if not self._check_status_response(resp.status, request_id): + return None + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict + del resp_dict['status'] + return resp_dict + return None + + def get_component_versions_json(self, search: dict) -> dict: + """ + Client function to call the rpc for Components GetComponentVersions + :param search: Message to send to the service + :return: Server response or None + """ + if not search: + self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + return None + request_id = str(uuid.uuid4()) + resp: CompVersionResponse + try: + request = ParseDict(search, CompVersionRequest()) # Parse the JSON/Dict into the purl request object + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + self.print_debug(f'Sending component version data (rqId: {request_id})...') + resp = self.comp_search_stub.GetComponentVersions(request, metadata=metadata, timeout=self.timeout) + except Exception as e: + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') + else: + if resp: + if not self._check_status_response(resp.status, request_id): + return None + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict del resp_dict['status'] return resp_dict return None diff --git a/tests/data/comp-search-input.json b/tests/data/comp-search-input.json new file mode 100644 index 00000000..fa02b463 --- /dev/null +++ b/tests/data/comp-search-input.json @@ -0,0 +1,5 @@ +{ + "search": "unoconv", + "package": "github", + "limit": 5 +} \ No newline at end of file diff --git a/tests/data/comp-versions-input.json b/tests/data/comp-versions-input.json new file mode 100644 index 00000000..9b7c95aa --- /dev/null +++ b/tests/data/comp-versions-input.json @@ -0,0 +1,4 @@ +{ + "purl": "pkg:github/unoconv/unoconv", + "limit": 10 +} \ No newline at end of file diff --git a/tests/data/package.json b/tests/data/package.json new file mode 100644 index 00000000..ff143968 --- /dev/null +++ b/tests/data/package.json @@ -0,0 +1,21 @@ +{ + "name": "vue", + "version": "2.6.12", + "description": "Reactive, component-oriented view layer for modern web interfaces.", + "main": "dist/vue.runtime.common.js", + "module": "dist/vue.runtime.esm.js", + "unpkg": "dist/vue.js", + "jsdelivr": "dist/vue.js", + "typings": "types/index.d.ts", + "files": [ + "src", + "dist/*.js", + "types/*.d.ts" + ], + "author": "Evan You", + "license": "MIT", + "homepage": "https://github.com/vuejs/vue#readme", + "devDependencies": { + "@babel/core": ">0.2.0" + } +} diff --git a/tests/data/purl-input-with-requirement.json b/tests/data/purl-input-with-requirement.json new file mode 100644 index 00000000..c7cfd716 --- /dev/null +++ b/tests/data/purl-input-with-requirement.json @@ -0,0 +1,8 @@ +{ + "purls": [ + { + "purl": "pkg:github/torvalds/linux", + "requirement": ">v5.13" + } + ] +} \ No newline at end of file diff --git a/tests/data/purl-input.json b/tests/data/purl-input.json new file mode 100644 index 00000000..93e8e2f3 --- /dev/null +++ b/tests/data/purl-input.json @@ -0,0 +1,7 @@ +{ + "purls": [ + { + "purl": "pkg:github/torvalds/linux@v5.13" + } + ] +} \ No newline at end of file From acf6f05b68b4be0e2801630efe0e8e0b449d93eb Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:11:39 +0100 Subject: [PATCH 122/489] Semgrep feature (#27) * added semgrep support * cleanup commented out code * added api key to command * fix help mistake * update to latest papi interface api code * added extra respose code designations * semgrep release * fixing dependency requirements --- CHANGELOG.md | 6 + CLIENT_HELP.md | 15 ++- requirements.txt | 3 +- setup.cfg | 1 + src/protoc_gen_swagger/__init__.py | 20 +++ src/protoc_gen_swagger/options/__init__.py | 20 +++ .../options/annotations_pb2.py | 31 +++++ .../options/annotations_pb2_grpc.py | 4 + .../options/openapiv2_pb2.py | 118 ++++++++++++++++++ .../options/openapiv2_pb2_grpc.py | 4 + src/scanoss/__init__.py | 2 +- .../components/v2/scanoss_components_pb2.py | 58 ++++++--- .../v2/scanoss_components_pb2_grpc.py | 46 +++++-- .../v2/scanoss_cryptography_pb2.py | 26 ++-- .../v2/scanoss_cryptography_pb2_grpc.py | 92 +------------- .../v2/scanoss_dependencies_pb2.py | 42 ++++--- .../v2/scanoss_dependencies_pb2_grpc.py | 92 +------------- .../api/scanning/v2/scanoss_scanning_pb2.py | 12 +- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 4 - src/scanoss/api/semgrep/__init__.py | 23 ++++ src/scanoss/api/semgrep/v2/__init__.py | 23 ++++ .../api/semgrep/v2/scanoss_semgrep_pb2.py | 41 ++++++ .../semgrep/v2/scanoss_semgrep_pb2_grpc.py | 108 ++++++++++++++++ .../v2/scanoss_vulnerabilities_pb2.py | 44 ++++--- .../v2/scanoss_vulnerabilities_pb2_grpc.py | 12 -- src/scanoss/cli.py | 45 ++++++- src/scanoss/components.py | 26 ++++ src/scanoss/scanossapi.py | 3 - src/scanoss/scanossgrpc.py | 40 +++++- 29 files changed, 669 insertions(+), 292 deletions(-) create mode 100644 src/protoc_gen_swagger/__init__.py create mode 100644 src/protoc_gen_swagger/options/__init__.py create mode 100644 src/protoc_gen_swagger/options/annotations_pb2.py create mode 100644 src/protoc_gen_swagger/options/annotations_pb2_grpc.py create mode 100644 src/protoc_gen_swagger/options/openapiv2_pb2.py create mode 100644 src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py create mode 100644 src/scanoss/api/semgrep/__init__.py create mode 100644 src/scanoss/api/semgrep/v2/__init__.py create mode 100644 src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py create mode 100644 src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ce0fbb..caed073f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.8.0] - 2023-11-13 +### Added +- Added Component Decoration sub-command: + - Semgrep (`scanoss-py comp semgrep`) + ## [1.7.0] - 2023-09-15 ### Added - Added Component Decoration sub-commands: @@ -269,3 +274,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.6.2]: https://github.com/scanoss/scanoss.py/compare/v1.6.1...v1.6.2 [1.6.3]: https://github.com/scanoss/scanoss.py/compare/v1.6.2...v1.6.3 [1.7.0]: https://github.com/scanoss/scanoss.py/compare/v1.6.3...v1.7.0 +[1.7.0]: https://github.com/scanoss/scanoss.py/compare/v1.7.0...v1.8.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 2d42a972..d2af6cab 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -201,11 +201,10 @@ It is possible to supply multiple PURLs by repeating the `-p pkg` option, or pro scanoss-py comp vulns -i purl-input.json -o vulnernable-comps.json ``` - #### Component Search The following command provides the capability to search the SCANOSS KB for an Open Source component: ```bash -scanoss-py comp search -s "unoconv" +scanoss-py comp search --key $SC_API_KEY -s "unoconv" ``` This command will search through different combinations to retrieve a proposed list of components (i.e. vendor/component, component, vendor, purl). @@ -213,7 +212,6 @@ It is also possible to search by component and vendor, while restricting the pac ```bash scanoss-py comp search --key $SC_API_KEY -c unoconv -v unoconv -p github ``` - **Note:** This sub-command requires a subscription to SCANOSS premium data. #### Component Versions @@ -221,7 +219,6 @@ The following command provides the capability to search the SCANOSS KB for versi ```bash scanoss-py comp versions --key $SC_API_KEY -p "pkg:github/unoconv/unoconv" ``` - **Note:** This sub-command requires a subscription to SCANOSS premium data. #### Cryptographic Algorithms @@ -233,6 +230,16 @@ It is possible to supply multiple PURLs by repeating the `-p pkg` option, or pro ```bash scanoss-py comp crypto --key $SC_API_KEY -i purl-input.json -o crypto-components.json ``` +**Note:** This sub-command requires a subscription to SCANOSS premium data. +#### Semgrep Issues/Findings +The following command provides the capability to search the SCANOSS KB for any semgrep issues detected in a specified component PURL: +```bash +scanoss-py comp semgrep --key $SC_API_KEY -p "pkg:github/spring-projects/spring-data-jpa" +``` +It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): +```bash +scanoss-py comp semgrep --key $SC_API_KEY -i purl-input.json -o semgrep-issues.json +``` **Note:** This sub-command requires a subscription to SCANOSS premium data. diff --git a/requirements.txt b/requirements.txt index 33783dd4..8ef40e42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ grpcio>1.42.0 protobuf>3.19.1 pypac urllib3 -pyOpenSSL \ No newline at end of file +pyOpenSSL +google-api-core \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index e4c40f75..e84568a2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,7 @@ install_requires = protobuf>3.19.1 pypac pyOpenSSL + google-api-core [options.extras_require] fast_winnowing = diff --git a/src/protoc_gen_swagger/__init__.py b/src/protoc_gen_swagger/__init__.py new file mode 100644 index 00000000..6b78cbea --- /dev/null +++ b/src/protoc_gen_swagger/__init__.py @@ -0,0 +1,20 @@ +""" + SPDX-License-Identifier: BSD-3-Clause + + Copyright (c) 2015, Gengo, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Gengo, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. +""" diff --git a/src/protoc_gen_swagger/options/__init__.py b/src/protoc_gen_swagger/options/__init__.py new file mode 100644 index 00000000..6b78cbea --- /dev/null +++ b/src/protoc_gen_swagger/options/__init__.py @@ -0,0 +1,20 @@ +""" + SPDX-License-Identifier: BSD-3-Clause + + Copyright (c) 2015, Gengo, Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Gengo, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. +""" diff --git a/src/protoc_gen_swagger/options/annotations_pb2.py b/src/protoc_gen_swagger/options/annotations_pb2.py new file mode 100644 index 00000000..c568f388 --- /dev/null +++ b/src/protoc_gen_swagger/options/annotations_pb2.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: protoc-gen-swagger/options/annotations.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 +from protoc_gen_swagger.options import openapiv2_pb2 as protoc__gen__swagger_dot_options_dot_openapiv2__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,protoc-gen-swagger/options/annotations.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a google/protobuf/descriptor.proto\x1a*protoc-gen-swagger/options/openapiv2.proto:j\n\x11openapiv2_swagger\x12\x1c.google.protobuf.FileOptions\x18\x92\x08 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Swagger:p\n\x13openapiv2_operation\x12\x1e.google.protobuf.MethodOptions\x18\x92\x08 \x01(\x0b\x32\x32.grpc.gateway.protoc_gen_swagger.options.Operation:k\n\x10openapiv2_schema\x12\x1f.google.protobuf.MessageOptions\x18\x92\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema:e\n\ropenapiv2_tag\x12\x1f.google.protobuf.ServiceOptions\x18\x92\x08 \x01(\x0b\x32,.grpc.gateway.protoc_gen_swagger.options.Tag:l\n\x0fopenapiv2_field\x12\x1d.google.protobuf.FieldOptions\x18\x92\x08 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchemaBCZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.annotations_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + google_dot_protobuf_dot_descriptor__pb2.FileOptions.RegisterExtension(openapiv2_swagger) + google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(openapiv2_operation) + google_dot_protobuf_dot_descriptor__pb2.MessageOptions.RegisterExtension(openapiv2_schema) + google_dot_protobuf_dot_descriptor__pb2.ServiceOptions.RegisterExtension(openapiv2_tag) + google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(openapiv2_field) + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' +# @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/annotations_pb2_grpc.py b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py new file mode 100644 index 00000000..2daafffe --- /dev/null +++ b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2.py b/src/protoc_gen_swagger/options/openapiv2_pb2.py new file mode 100644 index 00000000..0df96e43 --- /dev/null +++ b/src/protoc_gen_swagger/options/openapiv2_pb2.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: protoc-gen-swagger/options/openapiv2.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*protoc-gen-swagger/options/openapiv2.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xa0\x07\n\x07Swagger\x12\x0f\n\x07swagger\x18\x01 \x01(\t\x12;\n\x04info\x18\x02 \x01(\x0b\x32-.grpc.gateway.protoc_gen_swagger.options.Info\x12\x0c\n\x04host\x18\x03 \x01(\t\x12\x11\n\tbase_path\x18\x04 \x01(\t\x12O\n\x07schemes\x18\x05 \x03(\x0e\x32>.grpc.gateway.protoc_gen_swagger.options.Swagger.SwaggerScheme\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12R\n\tresponses\x18\n \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Swagger.ResponsesEntry\x12Z\n\x14security_definitions\x18\x0b \x01(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12U\n\rexternal_docs\x18\x0e \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12T\n\nextensions\x18\x0f \x03(\x0b\x32@.grpc.gateway.protoc_gen_swagger.options.Swagger.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"B\n\rSwaggerScheme\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04HTTP\x10\x01\x12\t\n\x05HTTPS\x10\x02\x12\x06\n\x02WS\x10\x03\x12\x07\n\x03WSS\x10\x04J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\r\x10\x0e\"\xa9\x05\n\tOperation\x12\x0c\n\x04tags\x18\x01 \x03(\t\x12\x0f\n\x07summary\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12U\n\rexternal_docs\x18\x04 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12\x14\n\x0coperation_id\x18\x05 \x01(\t\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12T\n\tresponses\x18\t \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Operation.ResponsesEntry\x12\x0f\n\x07schemes\x18\n \x03(\t\x12\x12\n\ndeprecated\x18\x0b \x01(\x08\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12V\n\nextensions\x18\r \x03(\x0b\x32\x42.grpc.gateway.protoc_gen_swagger.options.Operation.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01J\x04\x08\x08\x10\t\"\xab\x01\n\x06Header\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x06 \x01(\t\x12\x0f\n\x07pattern\x18\r \x01(\tJ\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\n\x10\x0bJ\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\rJ\x04\x08\x0e\x10\x0fJ\x04\x08\x0f\x10\x10J\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13\"\xb8\x04\n\x08Response\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12?\n\x06schema\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema\x12O\n\x07headers\x18\x03 \x03(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.Response.HeadersEntry\x12Q\n\x08\x65xamples\x18\x04 \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Response.ExamplesEntry\x12U\n\nextensions\x18\x05 \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Response.ExtensionsEntry\x1a_\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Header:\x02\x38\x01\x1a/\n\rExamplesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"\xf9\x02\n\x04Info\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x18\n\x10terms_of_service\x18\x03 \x01(\t\x12\x41\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Contact\x12\x41\n\x07license\x18\x05 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.License\x12\x0f\n\x07version\x18\x06 \x01(\t\x12Q\n\nextensions\x18\x07 \x03(\x0b\x32=.grpc.gateway.protoc_gen_swagger.options.Info.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"3\n\x07\x43ontact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\r\n\x05\x65mail\x18\x03 \x01(\t\"$\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"9\n\x15\x45xternalDocumentation\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"\x9c\x02\n\x06Schema\x12H\n\x0bjson_schema\x18\x01 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchema\x12\x15\n\rdiscriminator\x18\x02 \x01(\t\x12\x11\n\tread_only\x18\x03 \x01(\x08\x12U\n\rexternal_docs\x18\x05 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12)\n\x07\x65xample\x18\x06 \x01(\x0b\x32\x14.google.protobuf.AnyB\x02\x18\x01\x12\x16\n\x0e\x65xample_string\x18\x07 \x01(\tJ\x04\x08\x04\x10\x05\"\xe3\x05\n\nJSONSchema\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\t\x12\x11\n\tread_only\x18\x08 \x01(\x08\x12\x0f\n\x07\x65xample\x18\t \x01(\t\x12\x13\n\x0bmultiple_of\x18\n \x01(\x01\x12\x0f\n\x07maximum\x18\x0b \x01(\x01\x12\x19\n\x11\x65xclusive_maximum\x18\x0c \x01(\x08\x12\x0f\n\x07minimum\x18\r \x01(\x01\x12\x19\n\x11\x65xclusive_minimum\x18\x0e \x01(\x08\x12\x12\n\nmax_length\x18\x0f \x01(\x04\x12\x12\n\nmin_length\x18\x10 \x01(\x04\x12\x0f\n\x07pattern\x18\x11 \x01(\t\x12\x11\n\tmax_items\x18\x14 \x01(\x04\x12\x11\n\tmin_items\x18\x15 \x01(\x04\x12\x14\n\x0cunique_items\x18\x16 \x01(\x08\x12\x16\n\x0emax_properties\x18\x18 \x01(\x04\x12\x16\n\x0emin_properties\x18\x19 \x01(\x04\x12\x10\n\x08required\x18\x1a \x03(\t\x12\r\n\x05\x61rray\x18\" \x03(\t\x12W\n\x04type\x18# \x03(\x0e\x32I.grpc.gateway.protoc_gen_swagger.options.JSONSchema.JSONSchemaSimpleTypes\x12\x0e\n\x06\x66ormat\x18$ \x01(\t\x12\x0c\n\x04\x65num\x18. \x03(\t\"w\n\x15JSONSchemaSimpleTypes\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41RRAY\x10\x01\x12\x0b\n\x07\x42OOLEAN\x10\x02\x12\x0b\n\x07INTEGER\x10\x03\x12\x08\n\x04NULL\x10\x04\x12\n\n\x06NUMBER\x10\x05\x12\n\n\x06OBJECT\x10\x06\x12\n\n\x06STRING\x10\x07J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13J\x04\x08\x13\x10\x14J\x04\x08\x17\x10\x18J\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1dJ\x04\x08\x1d\x10\x1eJ\x04\x08\x1e\x10\"J\x04\x08%\x10*J\x04\x08*\x10+J\x04\x08+\x10.\"w\n\x03Tag\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12U\n\rexternal_docs\x18\x03 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentationJ\x04\x08\x01\x10\x02\"\xdd\x01\n\x13SecurityDefinitions\x12\\\n\x08security\x18\x01 \x03(\x0b\x32J.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions.SecurityEntry\x1ah\n\rSecurityEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.grpc.gateway.protoc_gen_swagger.options.SecurityScheme:\x02\x38\x01\"\x96\x06\n\x0eSecurityScheme\x12J\n\x04type\x18\x01 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Type\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x46\n\x02in\x18\x04 \x01(\x0e\x32:.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.In\x12J\n\x04\x66low\x18\x05 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Flow\x12\x19\n\x11\x61uthorization_url\x18\x06 \x01(\t\x12\x11\n\ttoken_url\x18\x07 \x01(\t\x12?\n\x06scopes\x18\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Scopes\x12[\n\nextensions\x18\t \x03(\x0b\x32G.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"K\n\x04Type\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\x0e\n\nTYPE_BASIC\x10\x01\x12\x10\n\x0cTYPE_API_KEY\x10\x02\x12\x0f\n\x0bTYPE_OAUTH2\x10\x03\"1\n\x02In\x12\x0e\n\nIN_INVALID\x10\x00\x12\x0c\n\x08IN_QUERY\x10\x01\x12\r\n\tIN_HEADER\x10\x02\"j\n\x04\x46low\x12\x10\n\x0c\x46LOW_INVALID\x10\x00\x12\x11\n\rFLOW_IMPLICIT\x10\x01\x12\x11\n\rFLOW_PASSWORD\x10\x02\x12\x14\n\x10\x46LOW_APPLICATION\x10\x03\x12\x14\n\x10\x46LOW_ACCESS_CODE\x10\x04\"\xc9\x02\n\x13SecurityRequirement\x12s\n\x14security_requirement\x18\x01 \x03(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementEntry\x1a)\n\x18SecurityRequirementValue\x12\r\n\x05scope\x18\x01 \x03(\t\x1a\x91\x01\n\x18SecurityRequirementEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x64\n\x05value\x18\x02 \x01(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementValue:\x02\x38\x01\"\x81\x01\n\x06Scopes\x12I\n\x05scope\x18\x01 \x03(\x0b\x32:.grpc.gateway.protoc_gen_swagger.options.Scopes.ScopeEntry\x1a,\n\nScopeEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x43ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.openapiv2_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' + _SWAGGER_RESPONSESENTRY._options = None + _SWAGGER_RESPONSESENTRY._serialized_options = b'8\001' + _SWAGGER_EXTENSIONSENTRY._options = None + _SWAGGER_EXTENSIONSENTRY._serialized_options = b'8\001' + _OPERATION_RESPONSESENTRY._options = None + _OPERATION_RESPONSESENTRY._serialized_options = b'8\001' + _OPERATION_EXTENSIONSENTRY._options = None + _OPERATION_EXTENSIONSENTRY._serialized_options = b'8\001' + _RESPONSE_HEADERSENTRY._options = None + _RESPONSE_HEADERSENTRY._serialized_options = b'8\001' + _RESPONSE_EXAMPLESENTRY._options = None + _RESPONSE_EXAMPLESENTRY._serialized_options = b'8\001' + _RESPONSE_EXTENSIONSENTRY._options = None + _RESPONSE_EXTENSIONSENTRY._serialized_options = b'8\001' + _INFO_EXTENSIONSENTRY._options = None + _INFO_EXTENSIONSENTRY._serialized_options = b'8\001' + _SCHEMA.fields_by_name['example']._options = None + _SCHEMA.fields_by_name['example']._serialized_options = b'\030\001' + _SECURITYDEFINITIONS_SECURITYENTRY._options = None + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_options = b'8\001' + _SECURITYSCHEME_EXTENSIONSENTRY._options = None + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_options = b'8\001' + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._options = None + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_options = b'8\001' + _SCOPES_SCOPEENTRY._options = None + _SCOPES_SCOPEENTRY._serialized_options = b'8\001' + _SWAGGER._serialized_start=145 + _SWAGGER._serialized_end=1073 + _SWAGGER_RESPONSESENTRY._serialized_start=813 + _SWAGGER_RESPONSESENTRY._serialized_end=912 + _SWAGGER_EXTENSIONSENTRY._serialized_start=914 + _SWAGGER_EXTENSIONSENTRY._serialized_end=987 + _SWAGGER_SWAGGERSCHEME._serialized_start=989 + _SWAGGER_SWAGGERSCHEME._serialized_end=1055 + _OPERATION._serialized_start=1076 + _OPERATION._serialized_end=1757 + _OPERATION_RESPONSESENTRY._serialized_start=813 + _OPERATION_RESPONSESENTRY._serialized_end=912 + _OPERATION_EXTENSIONSENTRY._serialized_start=914 + _OPERATION_EXTENSIONSENTRY._serialized_end=987 + _HEADER._serialized_start=1760 + _HEADER._serialized_end=1931 + _RESPONSE._serialized_start=1934 + _RESPONSE._serialized_end=2502 + _RESPONSE_HEADERSENTRY._serialized_start=2283 + _RESPONSE_HEADERSENTRY._serialized_end=2378 + _RESPONSE_EXAMPLESENTRY._serialized_start=2380 + _RESPONSE_EXAMPLESENTRY._serialized_end=2427 + _RESPONSE_EXTENSIONSENTRY._serialized_start=914 + _RESPONSE_EXTENSIONSENTRY._serialized_end=987 + _INFO._serialized_start=2505 + _INFO._serialized_end=2882 + _INFO_EXTENSIONSENTRY._serialized_start=914 + _INFO_EXTENSIONSENTRY._serialized_end=987 + _CONTACT._serialized_start=2884 + _CONTACT._serialized_end=2935 + _LICENSE._serialized_start=2937 + _LICENSE._serialized_end=2973 + _EXTERNALDOCUMENTATION._serialized_start=2975 + _EXTERNALDOCUMENTATION._serialized_end=3032 + _SCHEMA._serialized_start=3035 + _SCHEMA._serialized_end=3319 + _JSONSCHEMA._serialized_start=3322 + _JSONSCHEMA._serialized_end=4061 + _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_start=3864 + _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_end=3983 + _TAG._serialized_start=4063 + _TAG._serialized_end=4182 + _SECURITYDEFINITIONS._serialized_start=4185 + _SECURITYDEFINITIONS._serialized_end=4406 + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_start=4302 + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_end=4406 + _SECURITYSCHEME._serialized_start=4409 + _SECURITYSCHEME._serialized_end=5199 + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_start=914 + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_end=987 + _SECURITYSCHEME_TYPE._serialized_start=4965 + _SECURITYSCHEME_TYPE._serialized_end=5040 + _SECURITYSCHEME_IN._serialized_start=5042 + _SECURITYSCHEME_IN._serialized_end=5091 + _SECURITYSCHEME_FLOW._serialized_start=5093 + _SECURITYSCHEME_FLOW._serialized_end=5199 + _SECURITYREQUIREMENT._serialized_start=5202 + _SECURITYREQUIREMENT._serialized_end=5531 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_start=5342 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_end=5383 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_start=5386 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_end=5531 + _SCOPES._serialized_start=5534 + _SCOPES._serialized_end=5663 + _SCOPES_SCOPEENTRY._serialized_start=5619 + _SCOPES_SCOPEENTRY._serialized_end=5663 +# @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py new file mode 100644 index 00000000..2daafffe --- /dev/null +++ b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py @@ -0,0 +1,4 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 3ede3655..75dfd30d 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.7.0' +__version__ = '1.8.0' diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index ced1fc75..cf1290a9 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -12,32 +12,50 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xcb\x02\n\nComponents\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x12q\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"\x00\x12w\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"\x00\x42\x37Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05\"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xd4\x04\n\nComponents\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/components/echo:\x01*\x12\x95\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/components/search:\x01*\x12\x9d\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"&\x82\xd3\xe4\x93\x02 \"\x1b/api/v2/components/versions:\x01*\x12\x98\x01\n\x16GetComponentStatistics\x12\".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse\"(\x82\xd3\xe4\x93\x02\"\"\x1d/api/v2/components/statistics:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Components Service\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2' - _COMPSEARCHREQUEST._serialized_start=125 - _COMPSEARCHREQUEST._serialized_end=243 - _COMPSEARCHRESPONSE._serialized_start=246 - _COMPSEARCHRESPONSE._serialized_end=457 - _COMPSEARCHRESPONSE_COMPONENT._serialized_start=400 - _COMPSEARCHRESPONSE_COMPONENT._serialized_end=457 - _COMPVERSIONREQUEST._serialized_start=459 - _COMPVERSIONREQUEST._serialized_end=508 - _COMPVERSIONRESPONSE._serialized_start=511 - _COMPVERSIONRESPONSE._serialized_end=981 - _COMPVERSIONRESPONSE_LICENSE._serialized_start=666 - _COMPVERSIONRESPONSE_LICENSE._serialized_end=745 - _COMPVERSIONRESPONSE_VERSION._serialized_start=747 - _COMPVERSIONRESPONSE_VERSION._serialized_end=847 - _COMPVERSIONRESPONSE_COMPONENT._serialized_start=850 - _COMPVERSIONRESPONSE_COMPONENT._serialized_end=981 - _COMPONENTS._serialized_start=984 - _COMPONENTS._serialized_end=1315 + DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\331\001\022s\n\032SCANOSS Components Service\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _COMPONENTS.methods_by_name['Echo']._options = None + _COMPONENTS.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/components/echo:\001*' + _COMPONENTS.methods_by_name['SearchComponents']._options = None + _COMPONENTS.methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/components/search:\001*' + _COMPONENTS.methods_by_name['GetComponentVersions']._options = None + _COMPONENTS.methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002 \"\033/api/v2/components/versions:\001*' + _COMPONENTS.methods_by_name['GetComponentStatistics']._options = None + _COMPONENTS.methods_by_name['GetComponentStatistics']._serialized_options = b'\202\323\344\223\002\"\"\035/api/v2/components/statistics:\001*' + _COMPSEARCHREQUEST._serialized_start=201 + _COMPSEARCHREQUEST._serialized_end=319 + _COMPSTATISTIC._serialized_start=322 + _COMPSTATISTIC._serialized_end=524 + _COMPSTATISTIC_LANGUAGE._serialized_start=485 + _COMPSTATISTIC_LANGUAGE._serialized_end=524 + _COMPSTATISTICRESPONSE._serialized_start=527 + _COMPSTATISTICRESPONSE._serialized_end=778 + _COMPSTATISTICRESPONSE_PURLS._serialized_start=678 + _COMPSTATISTICRESPONSE_PURLS._serialized_end=778 + _COMPSEARCHRESPONSE._serialized_start=781 + _COMPSEARCHRESPONSE._serialized_end=992 + _COMPSEARCHRESPONSE_COMPONENT._serialized_start=935 + _COMPSEARCHRESPONSE_COMPONENT._serialized_end=992 + _COMPVERSIONREQUEST._serialized_start=994 + _COMPVERSIONREQUEST._serialized_end=1043 + _COMPVERSIONRESPONSE._serialized_start=1046 + _COMPVERSIONRESPONSE._serialized_end=1516 + _COMPVERSIONRESPONSE_LICENSE._serialized_start=1201 + _COMPVERSIONRESPONSE_LICENSE._serialized_end=1280 + _COMPVERSIONRESPONSE_VERSION._serialized_start=1282 + _COMPVERSIONRESPONSE_VERSION._serialized_end=1382 + _COMPVERSIONRESPONSE_COMPONENT._serialized_start=1385 + _COMPVERSIONRESPONSE_COMPONENT._serialized_end=1516 + _COMPONENTS._serialized_start=1519 + _COMPONENTS._serialized_end=2115 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py index 956d55d4..8ac7b92c 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py @@ -32,6 +32,11 @@ def __init__(self, channel): request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, ) + self.GetComponentStatistics = channel.unary_unary( + '/scanoss.api.components.v2.Components/GetComponentStatistics', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, + ) class ComponentsServicer(object): @@ -41,10 +46,6 @@ class ComponentsServicer(object): def Echo(self, request, context): """Standard echo - option (google.api.http) = { - post: "/api/v2/components/echo" - body: "*" - }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -52,10 +53,6 @@ def Echo(self, request, context): def SearchComponents(self, request, context): """Search for components - option (google.api.http) = { - post: "/api/v2/components/search" - body: "*" - }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -63,10 +60,13 @@ def SearchComponents(self, request, context): def GetComponentVersions(self, request, context): """Get all version information for a specific component - option (google.api.http) = { - post: "/api/v2/components/versions" - body: "*" - }; + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentStatistics(self, request, context): + """Get the statistics for the specified components """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -90,6 +90,11 @@ def add_ComponentsServicer_to_server(servicer, server): request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.FromString, response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.SerializeToString, ), + 'GetComponentStatistics': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentStatistics, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.components.v2.Components', rpc_method_handlers) @@ -152,3 +157,20 @@ def GetComponentVersions(request, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetComponentStatistics(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/GetComponentStatistics', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py index 2a84b5e6..3431e10e 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py @@ -12,22 +12,28 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xb9\x02\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x31\n\nAlgorithms\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\x1au\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12M\n\nalgorithms\x18\x03 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.AlgorithmResponse.Algorithms2\xc8\x01\n\x0c\x43ryptography\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x12\x65\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"\x00\x42;Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xb9\x02\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x31\n\nAlgorithms\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\x1au\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12M\n\nalgorithms\x18\x03 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.AlgorithmResponse.Algorithms2\x97\x02\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/cryptography/algorithms:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2' - _ALGORITHMRESPONSE._serialized_start=132 - _ALGORITHMRESPONSE._serialized_end=445 - _ALGORITHMRESPONSE_ALGORITHMS._serialized_start=277 - _ALGORITHMRESPONSE_ALGORITHMS._serialized_end=326 - _ALGORITHMRESPONSE_PURLS._serialized_start=328 - _ALGORITHMRESPONSE_PURLS._serialized_end=445 - _CRYPTOGRAPHY._serialized_start=448 - _CRYPTOGRAPHY._serialized_end=648 + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _CRYPTOGRAPHY.methods_by_name['Echo']._options = None + _CRYPTOGRAPHY.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/cryptography/echo:\001*' + _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._options = None + _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/cryptography/algorithms:\001*' + _ALGORITHMRESPONSE._serialized_start=208 + _ALGORITHMRESPONSE._serialized_end=521 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_start=353 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_end=402 + _ALGORITHMRESPONSE_PURLS._serialized_start=404 + _ALGORITHMRESPONSE_PURLS._serialized_end=521 + _CRYPTOGRAPHY._serialized_start=524 + _CRYPTOGRAPHY._serialized_end=803 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py index 18e23658..1d641e82 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py @@ -7,33 +7,7 @@ class CryptographyStub(object): - """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { - info: { - title: "SCANOSS Cryptography Service"; - version: "2.0"; - contact: { - name: "scanoss-cryptography"; - url: "https://github.com/scanoss/crpytography"; - email: "support@scanoss.com"; - }; - }; - schemes: HTTP; - consumes: "application/json"; - produces: "application/json"; - responses: { - key: "404"; - value: { - description: "Returned when the resource does not exist."; - schema: { - json_schema: { - type: STRING; - } - } - } - } - }; - - + """ Expose all of the SCANOSS Cryptography RPCs here """ @@ -56,42 +30,12 @@ def __init__(self, channel): class CryptographyServicer(object): - """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { - info: { - title: "SCANOSS Cryptography Service"; - version: "2.0"; - contact: { - name: "scanoss-cryptography"; - url: "https://github.com/scanoss/crpytography"; - email: "support@scanoss.com"; - }; - }; - schemes: HTTP; - consumes: "application/json"; - produces: "application/json"; - responses: { - key: "404"; - value: { - description: "Returned when the resource does not exist."; - schema: { - json_schema: { - type: STRING; - } - } - } - } - }; - - + """ Expose all of the SCANOSS Cryptography RPCs here """ def Echo(self, request, context): """Standard echo - option (google.api.http) = { - post: "/api/v2/cryptography/echo" - body: "*" - }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -99,10 +43,6 @@ def Echo(self, request, context): def GetAlgorithms(self, request, context): """Get Cryptographic algorithms associated with a list of PURLs - option (google.api.http) = { - post: "/api/v2/cryptography/algorithms" - body: "*" - }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -129,33 +69,7 @@ def add_CryptographyServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class Cryptography(object): - """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { - info: { - title: "SCANOSS Cryptography Service"; - version: "2.0"; - contact: { - name: "scanoss-cryptography"; - url: "https://github.com/scanoss/crpytography"; - email: "support@scanoss.com"; - }; - }; - schemes: HTTP; - consumes: "application/json"; - produces: "application/json"; - responses: { - key: "404"; - value: { - description: "Returned when the resource does not exist."; - schema: { - json_schema: { - type: STRING; - } - } - } - } - }; - - + """ Expose all of the SCANOSS Cryptography RPCs here """ diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index b0613fe8..0090b4cd 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -12,30 +12,36 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xd7\x01\n\x0c\x44\x65pendencies\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x12t\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\"\x00\x42;Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xa8\x02\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2' - _DEPENDENCYREQUEST._serialized_start=132 - _DEPENDENCYREQUEST._serialized_end=371 - _DEPENDENCYREQUEST_PURLS._serialized_start=237 - _DEPENDENCYREQUEST_PURLS._serialized_end=279 - _DEPENDENCYREQUEST_FILES._serialized_start=281 - _DEPENDENCYREQUEST_FILES._serialized_end=371 - _DEPENDENCYRESPONSE._serialized_start=374 - _DEPENDENCYRESPONSE._serialized_end=910 - _DEPENDENCYRESPONSE_LICENSES._serialized_start=521 - _DEPENDENCYRESPONSE_LICENSES._serialized_end=601 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=604 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=774 - _DEPENDENCYRESPONSE_FILES._serialized_start=777 - _DEPENDENCYRESPONSE_FILES._serialized_end=910 - _DEPENDENCIES._serialized_start=913 - _DEPENDENCIES._serialized_end=1128 + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service\"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _DEPENDENCIES.methods_by_name['Echo']._options = None + _DEPENDENCIES.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' + _DEPENDENCIES.methods_by_name['GetDependencies']._options = None + _DEPENDENCIES.methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' + _DEPENDENCYREQUEST._serialized_start=208 + _DEPENDENCYREQUEST._serialized_end=447 + _DEPENDENCYREQUEST_PURLS._serialized_start=313 + _DEPENDENCYREQUEST_PURLS._serialized_end=355 + _DEPENDENCYREQUEST_FILES._serialized_start=357 + _DEPENDENCYREQUEST_FILES._serialized_end=447 + _DEPENDENCYRESPONSE._serialized_start=450 + _DEPENDENCYRESPONSE._serialized_end=986 + _DEPENDENCYRESPONSE_LICENSES._serialized_start=597 + _DEPENDENCYRESPONSE_LICENSES._serialized_end=677 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=680 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=850 + _DEPENDENCYRESPONSE_FILES._serialized_start=853 + _DEPENDENCYRESPONSE_FILES._serialized_end=986 + _DEPENDENCIES._serialized_start=989 + _DEPENDENCIES._serialized_end=1285 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py index 25c631ff..cddf7cfa 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py @@ -7,33 +7,7 @@ class DependenciesStub(object): - """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { - info: { - title: "SCANOSS Dependency Service"; - version: "2.0"; - contact: { - name: "scanoss-dependencies"; - url: "https://github.com/scanoss/dependencies"; - email: "support@scanoss.com"; - }; - }; - schemes: HTTP; - consumes: "application/json"; - produces: "application/json"; - responses: { - key: "404"; - value: { - description: "Returned when the resource does not exist."; - schema: { - json_schema: { - type: STRING; - } - } - } - } - }; - - + """ Expose all of the SCANOSS Dependency RPCs here """ @@ -56,42 +30,12 @@ def __init__(self, channel): class DependenciesServicer(object): - """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { - info: { - title: "SCANOSS Dependency Service"; - version: "2.0"; - contact: { - name: "scanoss-dependencies"; - url: "https://github.com/scanoss/dependencies"; - email: "support@scanoss.com"; - }; - }; - schemes: HTTP; - consumes: "application/json"; - produces: "application/json"; - responses: { - key: "404"; - value: { - description: "Returned when the resource does not exist."; - schema: { - json_schema: { - type: STRING; - } - } - } - } - }; - - + """ Expose all of the SCANOSS Dependency RPCs here """ def Echo(self, request, context): """Standard echo - option (google.api.http) = { - post: "/api/v2/dependencies/echo" - body: "*" - }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -99,10 +43,6 @@ def Echo(self, request, context): def GetDependencies(self, request, context): """Get dependency details - option (google.api.http) = { - post: "/api/v2/dependencies/dependencies" - body: "*" - }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -129,33 +69,7 @@ def add_DependenciesServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class Dependencies(object): - """option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { - info: { - title: "SCANOSS Dependency Service"; - version: "2.0"; - contact: { - name: "scanoss-dependencies"; - url: "https://github.com/scanoss/dependencies"; - email: "support@scanoss.com"; - }; - }; - schemes: HTTP; - consumes: "application/json"; - produces: "application/json"; - responses: { - key: "404"; - value: { - description: "Returned when the resource does not exist."; - schema: { - json_schema: { - type: STRING; - } - } - } - } - }; - - + """ Expose all of the SCANOSS Dependency RPCs here """ diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 0ccff734..60c795dd 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -12,16 +12,20 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto2]\n\x08Scanning\x12Q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x00\x42\x33Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto2}\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2' - _SCANNING._serialized_start=119 - _SCANNING._serialized_end=212 + DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service\"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _SCANNING.methods_by_name['Echo']._options = None + _SCANNING.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' + _SCANNING._serialized_start=195 + _SCANNING._serialized_end=320 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index 864c1bd6..f6530e94 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -28,10 +28,6 @@ class ScanningServicer(object): def Echo(self, request, context): """Standard echo - option (google.api.http) = { - post: "/api/v2/scanning/echo" - body: "*" - }; """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/src/scanoss/api/semgrep/__init__.py b/src/scanoss/api/semgrep/__init__.py new file mode 100644 index 00000000..31c0cbaa --- /dev/null +++ b/src/scanoss/api/semgrep/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2023, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/semgrep/v2/__init__.py b/src/scanoss/api/semgrep/v2/__init__.py new file mode 100644 index 00000000..31c0cbaa --- /dev/null +++ b/src/scanoss/api/semgrep/v2/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2023, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py new file mode 100644 index 00000000..1b8b0461 --- /dev/null +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: scanoss/api/semgrep/v2/scanoss-semgrep.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12\".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse\"!\x82\xd3\xe4\x93\x02\x1b\"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service\"J\n\x0fscanoss-semgrep\x12\"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.semgrep.v2.scanoss_semgrep_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service\"J\n\017scanoss-semgrep\022\"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _SEMGREP.methods_by_name['Echo']._options = None + _SEMGREP.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\"\024/api/v2/semgrep/echo:\001*' + _SEMGREP.methods_by_name['GetIssues']._options = None + _SEMGREP.methods_by_name['GetIssues']._serialized_options = b'\202\323\344\223\002\033\"\026/api/v2/semgrep/issues:\001*' + _SEMGREPRESPONSE._serialized_start=193 + _SEMGREPRESPONSE._serialized_end=599 + _SEMGREPRESPONSE_ISSUE._serialized_start=329 + _SEMGREPRESPONSE_ISSUE._serialized_end=396 + _SEMGREPRESPONSE_FILE._serialized_start=398 + _SEMGREPRESPONSE_FILE._serialized_end=498 + _SEMGREPRESPONSE_PURLS._serialized_start=500 + _SEMGREPRESPONSE_PURLS._serialized_end=599 + _SEMGREP._serialized_start=602 + _SEMGREP._serialized_end=850 +# @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py new file mode 100644 index 00000000..4748a3ee --- /dev/null +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py @@ -0,0 +1,108 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from scanoss.api.semgrep.v2 import scanoss_semgrep_pb2 as scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2 + + +class SemgrepStub(object): + """ + Expose all of the SCANOSS Cryptography RPCs here + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Echo = channel.unary_unary( + '/scanoss.api.semgrep.v2.Semgrep/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) + self.GetIssues = channel.unary_unary( + '/scanoss.api.semgrep.v2.Semgrep/GetIssues', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, + ) + + +class SemgrepServicer(object): + """ + Expose all of the SCANOSS Cryptography RPCs here + """ + + def Echo(self, request, context): + """Standard echo + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetIssues(self, request, context): + """Get Potential issues associated with a list of PURLs + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_SemgrepServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetIssues': grpc.unary_unary_rpc_method_handler( + servicer.GetIssues, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Semgrep(object): + """ + Expose all of the SCANOSS Cryptography RPCs here + """ + + @staticmethod + def Echo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/Echo', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetIssues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/GetIssues', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index c041a460..9fc87ed3 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -12,30 +12,38 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n None: help='Retreive cryptographic algorithms for the given components') c_crypto.set_defaults(func=comp_crypto) + # Component Sub-command: component vulns c_vulns = comp_sub.add_parser('vulns', aliases=['vulnerabilities', 'vu'], description=f'Show Vulnerability details: {__version__}', help='Retreive vulnerabilities for the given components') c_vulns.set_defaults(func=comp_vulns) + # Component Sub-command: component semgrep + c_semgrep = comp_sub.add_parser('semgrep', aliases=['sp'], + description=f'Show Semgrep findings: {__version__}', + help='Retreive semgrep issues/findings for the given components') + c_semgrep.set_defaults(func=comp_semgrep) + + # Component Sub-command: component search c_search = comp_sub.add_parser('search', aliases=['sc'], description=f'Search component details: {__version__}', help='Search for a KB component') @@ -187,6 +195,7 @@ def setup_args() -> None: c_search.add_argument('--offset', '-f', type=int, help='Generic component search') c_search.set_defaults(func=comp_search) + # Component Sub-command: component versions c_versions = comp_sub.add_parser('versions', aliases=['vs'], description=f'Get component version details: {__version__}', help='Search for component versions') @@ -196,11 +205,11 @@ def setup_args() -> None: c_versions.set_defaults(func=comp_versions) # Common purl Component sub-command options - for p in [c_crypto, c_vulns]: + for p in [c_crypto, c_vulns, c_semgrep]: p.add_argument('--purl', '-p', type=str, nargs="*", help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') # Common Component sub-command options - for p in [c_crypto, c_vulns, c_search, c_versions]: + for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p.add_argument('--timeout', '-M', type=int, default=600, help='Timeout (in seconds) for API communication (optional - default 600)') @@ -252,7 +261,7 @@ def setup_args() -> None: p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') # Global Scan/GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: p.add_argument('--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)') p.add_argument('--proxy', type=str, help='Proxy URL to use for connections (optional). ' @@ -266,14 +275,15 @@ def setup_args() -> None: '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC') # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: p.add_argument('--api2url', type=str, - help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)') + help='SCANOSS gRPC API 2.0 URL (optional - default: https://osskb.org)') p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' 'Can also use the environment variable "grcp_proxy=:"') # Help/Trace command options - for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto, c_vulns, c_search, c_versions]: + for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto, c_vulns, c_search, + c_versions, c_semgrep]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -767,6 +777,29 @@ def comp_vulns(parser, args): if not comps.get_vulnerabilities(args.input, args.purl, args.output): exit(1) +def comp_semgrep(parser, args): + """ + Run the "component semgrep" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + exit(1) + pac_file = get_pac_file(args.pac) + comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + timeout=args.timeout) + if not comps.get_semgrep_details(args.input, args.purl, args.output): + exit(1) def comp_search(parser, args): """ diff --git a/src/scanoss/components.py b/src/scanoss/components.py index c7993c77..705bf04c 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -193,6 +193,32 @@ def get_vulnerabilities(self, json_file: str = None, purls: [] = None, output_fi self._close_file(output_file, file) return success + def get_semgrep_details(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: + """ + Retrieve the semgrep details for the supplied PURLs + + :param json_file: PURL JSON request file (optional) + :param purls: PURL request array (optional) + :param output_file: output filename (optional). Default: STDOUT + :return: True on success, False otherwise + """ + success = False + purls_request = self.load_purls(json_file, purls) + if purls_request is None or len(purls_request) == 0: + return False + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + self.print_msg('Sending PURLs to Semgrep API for decoration...') + response = self.grpc_api.get_semgrep_json(purls_request) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success + def search_components(self, output_file: str = None, json_file: str = None, search: str = None, vendor: str = None, comp: str = None, package: str = None, limit: int = None, offset: int = None) -> bool: diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 9f4248aa..ad68fb63 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -157,9 +157,6 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): r = self.session.post(self.url, files=scan_files, data=form_data, headers=self.headers, timeout=self.timeout ) - # r = requests.post(self.url, files=scan_files, data=form_data, headers=self.headers, - # timeout=self.timeout, verify=self.verify, proxies=self.proxies - # ) except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e: self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 42945248..d0251cc4 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -37,10 +37,12 @@ from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub +from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, DependencyResponse from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse +from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse from .api.components.v2.scanoss_components_pb2 import CompSearchRequest, CompSearchResponse, CompVersionRequest, CompVersionResponse from .scanossbase import ScanossBase from . import __version__ @@ -108,6 +110,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu self.comp_search_stub = ComponentsStub(grpc.insecure_channel(self.url)) self.crypto_stub = CryptographyStub(grpc.insecure_channel(self.url)) self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) + self.semgrep_stub = SemgrepStub(grpc.insecure_channel(self.url)) self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url)) else: if ca_cert is not None: @@ -117,6 +120,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu self.comp_search_stub = ComponentsStub(grpc.secure_channel(self.url, credentials)) self.crypto_stub = CryptographyStub(grpc.secure_channel(self.url, credentials)) self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) + self.semgrep_stub = SemgrepStub(grpc.secure_channel(self.url, credentials)) self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials)) @classmethod @@ -279,6 +283,35 @@ def get_vulnerabilities_json(self, purls: dict) -> dict: return resp_dict return None + def get_semgrep_json(self, purls: dict) -> dict: + """ + Client function to call the rpc for Semgrep GetIssues + :param purls: Message to send to the service + :return: Server response or None + """ + if not purls: + self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + return None + request_id = str(uuid.uuid4()) + resp: SemgrepResponse + try: + request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + self.print_debug(f'Sending semgrep data for decoration (rqId: {request_id})...') + resp = self.semgrep_stub.GetIssues(request, metadata=metadata, timeout=self.timeout) + except Exception as e: + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') + else: + if resp: + if not self._check_status_response(resp.status, request_id): + return None + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict + del resp_dict['status'] + return resp_dict + return None + def search_components_json(self, search: dict) -> dict: """ Client function to call the rpc for Components SearchComponents @@ -349,7 +382,12 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st self.print_debug(f'Checking response status (rqId: {request_id}): {status_response}') status_code: StatusCode = status_response.status if status_code > 1: - self.print_stderr(f'Not such a success (rqId: {request_id}): {status_response.message}') + msg = "Unsuccessful" + if status_code == 2: + msg = "Succeeded with warnings" + elif status_code == 3: + msg = "Failed with warnings" + self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}') return False return True From a2325910f6c6ff0831dd9aa4f67e246f0574386c Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Tue, 2 Jan 2024 23:14:25 +0100 Subject: [PATCH 123/489] Scan Dependency Decoration (#28) * added support for dependency decoration as part of scanning * added support for dependency decoration as part of scanning --- CHANGELOG.md | 8 +++++- CLIENT_HELP.md | 22 ++++++++++++++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 24 ++++++++++------- src/scanoss/scancodedeps.py | 20 ++++++++++++++ src/scanoss/scanner.py | 41 ++++++++++++++++++++++++----- src/scanoss/threadeddependencies.py | 32 +++++++++++++++------- 7 files changed, 122 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index caed073f..9aaea4e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.9.0] - 2023-12-29 +### Added +- Added dependency file decoration option to scanning (`scan`) using `--dep` + - More details can be found in [CLIENT_HELP.md](CLIENT_HELP.md) + ## [1.8.0] - 2023-11-13 ### Added - Added Component Decoration sub-command: @@ -274,4 +279,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.6.2]: https://github.com/scanoss/scanoss.py/compare/v1.6.1...v1.6.2 [1.6.3]: https://github.com/scanoss/scanoss.py/compare/v1.6.2...v1.6.3 [1.7.0]: https://github.com/scanoss/scanoss.py/compare/v1.6.3...v1.7.0 -[1.7.0]: https://github.com/scanoss/scanoss.py/compare/v1.7.0...v1.8.0 +[1.8.0]: https://github.com/scanoss/scanoss.py/compare/v1.7.0...v1.8.0 +[1.9.0]: https://github.com/scanoss/scanoss.py/compare/v1.8.0...v1.9.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index d2af6cab..e0131616 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -156,6 +156,22 @@ This fingerprint (WFP) can then be sent to the SCANOSS engine using the scanning scanoss-py scan -w src-fingers.wfp -o scan-results.json ``` +### Dependency file parsing +The dependency files of a project can be fingerprinted/parsed using the `dep` command: +```bash +scanoss-py dep -o src-deps.json src +``` + +This parsed dependency file can then be sent to the SCANOSS for decoration using the scanning command: +```bash +scanoss-py scan --dep src-deps.json --dependencies-only -o scan-results.json +``` + +It is possible to combine a WFP & Dependency file into a single scan also: +```bash +scanoss-py scan -w src-fingers.wfp --dep src-deps.json -o scan-results.json +``` + ### Scan a project folder The following command provides the capability to scan a given file/folder: ```bash @@ -167,6 +183,12 @@ The following command scans the `src` folder and writes the output to `scan-resu scanoss-py scan -o scan-results.json src ``` +### Scan a project folder with dependencies +The following command scans the `src` folder file, snippet & dependency matches, writing the output to `scan-results.json`: +```bash +scanoss-py scan -o scan-results.json -D src +``` + ### Converting RAW results into other formats The following command provides the capability to convert the RAW scan results from a SCANOSS scan into multiple different formats, including CycloneDX, SPDX Lite, CSV, etc. For the full set of formats, please run: diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 75dfd30d..9273caa6 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.8.0' +__version__ = '1.9.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 4b8b1fa1..36a41f5f 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -406,7 +406,7 @@ def get_scan_options(args): scan_dependencies = 0 if args.skip_snippets: scan_snippets = 0 - if args.dependencies: + if args.dependencies or args.dep: scan_dependencies = ScanType.SCAN_DEPENDENCIES.value if args.dependencies_only: scan_files = scan_snippets = 0 @@ -437,8 +437,8 @@ def scan(parser, args): args: Namespace Parsed arguments """ - if not args.scan_dir and not args.wfp and not args.stdin: - print_stderr('Please specify a file/folder, fingerprint (--wfp) or STDIN (--stdin)') + if not args.scan_dir and not args.wfp and not args.stdin and not args.dep: + print_stderr('Please specify a file/folder, fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)') parser.parse_args([args.subparser, '-h']) exit(1) if args.pac and args.proxy: @@ -536,10 +536,10 @@ def scan(parser, args): if not scanner.is_file_or_snippet_scan(): print_stderr(f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})') exit(1) - if args.threads > 1: - scanner.scan_wfp_file_threaded(args.wfp) - else: - scanner.scan_wfp_file(args.wfp) + if scanner.is_dependency_scan() and not args.dep: + print_stderr(f'Error: Cannot specify WFP & Dependency scanning without a dependency file ({--dep})') + exit(1) + scanner.scan_wfp_with_options(args.wfp, args.dep) elif args.stdin: contents = sys.stdin.buffer.read() if not scanner.scan_contents(args.stdin, contents): @@ -549,14 +549,20 @@ def scan(parser, args): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') exit(1) if os.path.isdir(args.scan_dir): - if not scanner.scan_folder_with_options(args.scan_dir, scanner.winnowing.file_map): + if not scanner.scan_folder_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map): exit(1) elif os.path.isfile(args.scan_dir): - if not scanner.scan_file_with_options(args.scan_dir, scanner.winnowing.file_map): + if not scanner.scan_file_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map): exit(1) else: print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') exit(1) + elif args.dep: + if not args.dependencies_only: + print_stderr(f'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.') + exit(1) + if not scanner.scan_folder_with_options(".", args.dep, scanner.winnowing.file_map): + exit(1) else: print_stderr('No action found to process') exit(1) diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index e3e3dca7..e1b1bebf 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -215,6 +215,26 @@ def run_scan(self, output_file: str = None, what_to_scan: str = None) -> bool: self.print_stderr(f'ERROR: Issue running scancode dependency scan on {what_to_scan}: {e}') return False return True + + def load_from_file(self, json_file: str = None) -> json: + """ + Load the parsed JSON dependencies file and return the json object + :param json_file: dependency json file + :return: SCANOSS dependency JSON + """ + if not json_file: + self.print_stderr('ERROR: No parsed JSON file provided to load.') + return None + if not os.path.isfile(json_file): + self.print_stderr(f'ERROR: parsed JSON file does not exist or is not a file: {json_file}') + return None + with open(json_file, 'r') as f: + try: + return json.loads(f.read()) + except Exception as e: + self.print_stderr(f'ERROR: Problem loading input JSON: {e}') + return None + # # End of ScancodeDeps Class # diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 62c627de..e2f6e93e 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -313,10 +313,11 @@ def is_dependency_scan(self): return True return False - def scan_folder_with_options(self, scan_dir: str, file_map: dict = None) -> bool: + def scan_folder_with_options(self, scan_dir: str, deps_file: str = None, file_map: dict = None) -> bool: """ Scan the given folder for whatever scaning options that have been configured :param scan_dir: directory to scan + :param deps_file: pre-parsed dependency file to decorate :param file_map: mapping of obfuscated files back into originals :return: True if successful, False otherwise """ @@ -331,7 +332,7 @@ def scan_folder_with_options(self, scan_dir: str, file_map: dict = None) -> bool if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') if self.is_dependency_scan(): - if not self.threaded_deps.run(what_to_scan=scan_dir, wait=False): # Kick off a background dependency scan + if not self.threaded_deps.run(what_to_scan=scan_dir, deps_file=deps_file, wait=False): # Kick off a background dependency scan success = False if self.is_file_or_snippet_scan(): if not self.scan_folder(scan_dir): @@ -542,10 +543,11 @@ def __finish_scan_threaded(self, file_map: dict = None) -> bool: success = False return success - def scan_file_with_options(self, file: str, file_map: dict = None) -> bool: + def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dict = None) -> bool: """ Scan the given file for whatever scaning options that have been configured :param file: file to scan + :param deps_file: pre-parsed dependency file to decorate :param file_map: mapping of obfuscated files back into originals :return: True if successful, False otherwise """ @@ -560,7 +562,7 @@ def scan_file_with_options(self, file: str, file_map: dict = None) -> bool: if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') if self.is_dependency_scan(): - if not self.threaded_deps.run(what_to_scan=file, wait=False): # Kick off a background dependency scan + if not self.threaded_deps.run(what_to_scan=file, deps_file=deps_file, wait=False): # Kick off a background dependency scan success = False if self.is_file_or_snippet_scan(): if not self.scan_file(file): @@ -725,6 +727,35 @@ def scan_wfp_file(self, file: str = None) -> bool: return success + def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) -> bool: + """ + Scan the given WFP file for whatever scaning options that have been configured + :param wfp: WFP file to scan + :param deps_file: pre-parsed dependency file to decorate + :param file_map: mapping of obfuscated files back into originals + :return: True if successful, False otherwise + """ + success = True + wfp_file = wfp if wfp else self.wfp # If a WFP file is specified, use it, otherwise us the default + if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): + raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") + + if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): + raise Exception(f"ERROR: No scan options defined to scan folder: {scan_dir}") + + if self.scan_output: + self.print_msg(f'Writing results to {self.scan_output}...') + if self.is_dependency_scan(): + if not self.threaded_deps.run(deps_file=deps_file, wait=False): # Kick off a background dependency scan + success = False + if self.is_file_or_snippet_scan(): + if not self.scan_wfp_file_threaded(wfp_file, file_map): + success = False + if self.threaded_scan: + if not self.__finish_scan_threaded(file_map): + success = False + return success + def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> bool: """ Scan the contents of the specified WFP file (threaded) @@ -778,8 +809,6 @@ def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> boo if not self.__run_scan_threaded(scan_started, file_count): success = False - elif not self.__finish_scan_threaded(file_map): - success = False return success def scan_wfp(self, wfp: str) -> bool: diff --git a/src/scanoss/threadeddependencies.py b/src/scanoss/threadeddependencies.py index c6ca12b7..43463eb6 100644 --- a/src/scanoss/threadeddependencies.py +++ b/src/scanoss/threadeddependencies.py @@ -31,6 +31,8 @@ from .scanossbase import ScanossBase from .scanossgrpc import ScanossGrpc +DEP_FILE_PREFIX = "file=" # Default prefix to signify an existing parsed dependency file + @dataclass class ThreadedDependencies(ScanossBase): @@ -64,18 +66,23 @@ def responses(self) -> Dict: return resp return None - def run(self, what_to_scan: str = None, wait: bool = True) -> bool: + def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True) -> bool: """ Initiate a background scan for the specified file/dir :param what_to_scan: file/folder to scan + :param deps_file: file to decorate instead of scan (overrides what_to_scan option) :param wait: wait for completion :return: True if successful, False if error encountered """ what_to_scan = what_to_scan if what_to_scan else self.what_to_scan self._errors = False try: - self.print_msg(f'Searching {what_to_scan} for dependencies...') - self.inputs.put(what_to_scan) # Set up an input queue to enable the parent to wait for completion + if deps_file: # Decorate the given dependencies file + self.print_msg(f'Decorating {deps_file} dependencies...') + self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}') # Add to queue and have parent wait on it + else: # Search for dependencies to decorate + self.print_msg(f'Searching {what_to_scan} for dependencies...') + self.inputs.put(what_to_scan) # Add to queue and have parent wait on it self._thread = threading.Thread(target=self.scan_dependencies, daemon=True) self._thread.start() except Exception as e: @@ -87,22 +94,27 @@ def run(self, what_to_scan: str = None, wait: bool = True) -> bool: def scan_dependencies(self) -> None: """ - Scan for dependencies from the given file/dir (from the input queue) + Scan for dependencies from the given file/dir or from an input file (from the input queue). """ current_thread = threading.get_ident() self.print_trace(f'Starting dependency worker {current_thread}...') try: - what_to_scan = self.inputs.get(timeout=5) # Begin processing the dependency request - if not self.sc_deps.run_scan(what_to_scan=what_to_scan): - self._errors = True - else: - deps = self.sc_deps.produce_from_file() + what_to_scan = self.inputs.get(timeout=5) # Begin processing the dependency request + deps = None + if what_to_scan.startswith(DEP_FILE_PREFIX): # We have a pre-parsed dependency file, load it + deps = self.sc_deps.load_from_file(what_to_scan.strip(DEP_FILE_PREFIX)) + else: # Search the file/folder for dependency files to parse + if not self.sc_deps.run_scan(what_to_scan=what_to_scan): + self._errors = True + else: + deps = self.sc_deps.produce_from_file() + if not self._errors: if deps is None: self.print_stderr(f'Problem searching for dependencies for: {what_to_scan}') self._errors = True elif not deps: self.print_trace(f'No dependencies found to decorate for: {what_to_scan}') - else: # TODO add API call to get dep data + else: decorated_deps = self.grpc_api.get_dependencies(deps) if decorated_deps: self.output.put(decorated_deps) From 6afc5f8c72e6d3ff33bf799280b6bf6eb7c376d0 Mon Sep 17 00:00:00 2001 From: Alexander Gschrei Date: Thu, 1 Feb 2024 11:05:50 +0100 Subject: [PATCH 124/489] feat: add support for scanning a list of files (#29) --- src/scanoss/scanner.py | 111 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index e2f6e93e..b3e8cd1c 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -598,6 +598,117 @@ def scan_file(self, file: str) -> bool: success = False return success + def scan_files(self, files: list[str]) -> bool: + """ + Scan the specified list of files, producing fingerprints, send to the SCANOSS API and return results + Please note that by providing an explicit list you bypass any exclusions that may be defined on the scanner + :param files: list[str] + List of filenames to scan + :return True if successful, False otherwise + """ + success = True + if not files: + raise Exception(f"ERROR: Please provide a non-empty list of filenames to scan") + self.print_msg(f'Scanning {len(files)} files...') + spinner = None + if not self.quiet and self.isatty: + spinner = Spinner('Fingerprinting ') + save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan + wfp_list = [] + scan_block = '' + scan_size = 0 + queue_size = 0 + file_count = 0 # count all files fingerprinted + wfp_file_count = 0 # count number of files in each queue post + scan_started = False + for file in files: + if self.threaded_scan and self.threaded_scan.stop_scanning(): + self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') + break + f_size = 0 + try: + f_size = os.stat(file).st_size + except Exception as e: + self.print_trace( + f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink + if f_size > 0: # Ignore broken links and empty files + self.print_trace(f'Fingerprinting {file}...') + if spinner: + spinner.next() + wfp = self.winnowing.wfp_for_file(file, file) + if wfp is None or wfp == '': + self.print_stderr(f'Warning: No WFP returned for {path}') + continue + if save_wfps_for_print: + wfp_list.append(wfp) + file_count += 1 + if self.threaded_scan: + wfp_size = len(wfp.encode("utf-8")) + # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + scan_block += wfp + scan_size = len(scan_block.encode("utf-8")) + wfp_file_count += 1 + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do + scan_started = True + if not self.threaded_scan.run(wait=False): + self.print_stderr( + f'Warning: Some errors encounted while scanning. Results might be incomplete.') + success = False + # End for loop + if self.threaded_scan and scan_block != '': + self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted + if spinner: + spinner.finish() + + if file_count > 0: + if save_wfps_for_print: # Write a WFP file if no threading is requested + self.print_debug(f'Writing fingerprints to {self.wfp}') + with open(self.wfp, 'w') as f: + f.write(''.join(wfp_list)) + else: + self.print_debug(f'Skipping writing WFP file {self.wfp}') + if self.threaded_scan: + success = self.__run_scan_threaded(scan_started, file_count) + else: + Scanner.print_stderr(f'Warning: No files found to scan in folder: {scan_dir}') + return success + + def scan_files_with_options(self, files: list[str], deps_file: str = None, file_map: dict = None) -> bool: + """ + Scan the given folder for whatever scaning options that have been configured + :param files: list of files to scan + :param deps_file: pre-parsed dependency file to decorate + :param file_map: mapping of obfuscated files back into originals + :return: True if successful, False otherwise + """ + success = True + if not files: + raise Exception(f"ERROR: Please specify a list of files to scan") + if not self.is_file_or_snippet_scan(): + raise Exception(f"ERROR: file or snippet scan options have to be set to scan files: {files}") + if self.is_dependency_scan(): + raise Exception(f"ERROR: The dependency scan option is currently not supported when scanning a list of files") + if self.scan_output: + self.print_msg(f'Writing results to {self.scan_output}...') + if self.is_file_or_snippet_scan(): + if not self.scan_files(files): + success = False + if self.threaded_scan: + if not self.__finish_scan_threaded(file_map): + success = False + return success + def scan_contents(self, filename: str, contents: bytes) -> bool: """ Scan the given contents as a file From 4366284af28ac769cc77d3f5f4ed8f7519268a9c Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Fri, 9 Feb 2024 11:12:29 -0500 Subject: [PATCH 125/489] Client filters (#30) * do not report error when no dependency files found * added scan and wfp filter options * added WFP HPSM test --- .github/workflows/python-local-test.yml | 14 ++++++++ CHANGELOG.md | 10 ++++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 35 +++++++++--------- src/scanoss/scanner.py | 48 +++++++++++++++---------- src/scanoss/scanossgrpc.py | 3 +- src/scanoss/threadeddependencies.py | 4 +-- 7 files changed, 78 insertions(+), 38 deletions(-) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 6a375ca5..7ccc05eb 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -64,3 +64,17 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + + - name: Run Tests HPSM (fast winnowing) + run: | + pip install scanoss_winnowing + which scanoss-py + scanoss-py version + scanoss-py utils fast + scanoss-py wfp -H tests > fingers.wfp + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aaea4e5..eed0892e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.10.0] - 2024-02-09 +### Added +- Added scan/wfp file filtering options + - Exclude file extensions `--skip-extension` (repeat as needed) + - Exclude folder `--skip-folder` (repeat as needed) + - Exclude files smaller than specified `--skip-size` +- Added `scan_files_with_options` SDK capability + - Enables a programmer to supply a specific list of files to scan + ## [1.9.0] - 2023-12-29 ### Added - Added dependency file decoration option to scanning (`scan`) using `--dep` @@ -281,3 +290,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.7.0]: https://github.com/scanoss/scanoss.py/compare/v1.6.3...v1.7.0 [1.8.0]: https://github.com/scanoss/scanoss.py/compare/v1.7.0...v1.8.0 [1.9.0]: https://github.com/scanoss/scanoss.py/compare/v1.8.0...v1.9.0 +[1.10.0]: https://github.com/scanoss/scanoss.py/compare/v1.9.0...v1.10.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 9273caa6..151bffdf 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.9.0' +__version__ = '1.10.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 36a41f5f..9916c02d 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -86,7 +86,6 @@ def setup_args() -> None: '256: disable best match only, 512: hide identified files, ' '1024: enable download_url, 2048: enable GitHub full path, ' '4096: disable extended server stats)') - p_scan.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') p_scan.add_argument('--post-size', '-P', type=int, default=32, help='Number of kilobytes to limit the post to while scanning (optional - default 32)') p_scan.add_argument('--timeout', '-M', type=int, default=180, @@ -94,17 +93,12 @@ def setup_args() -> None: p_scan.add_argument('--retry', '-R', type=int, default=5, help='Retry limit for API communication (optional - default 5)') p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') - p_scan.add_argument('--all-extensions', action='store_true', help='Scan all file extensions') - p_scan.add_argument('--all-folders', action='store_true', help='Scan all folders') - p_scan.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') - p_scan.add_argument('--obfuscate', action='store_true', help='Obfuscate file paths and names') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') p_scan.add_argument('--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).') p_scan.add_argument('--sc-timeout', type=int, default=600, help='Timeout (in seconds) for scancode to complete (optional - default 600)') - p_scan.add_argument('--hpsm', '-H', action='store_true', help='Scan using High Precision Snippet Matching') # Sub-command: fingerprint p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'], @@ -116,12 +110,6 @@ def setup_args() -> None: p_wfp.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, help='Fingerprint the file contents supplied via STDIN (optional)') p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') - p_wfp.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') - p_wfp.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') - p_wfp.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions') - p_wfp.add_argument('--all-folders', action='store_true', help='Fingerprint all folders') - p_wfp.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders') - p_wfp.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.') # Sub-command: dependency p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], @@ -260,6 +248,19 @@ def setup_args() -> None: help='SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)') p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') + # Global Scan/Fingerprint filter options + for p in [p_scan, p_wfp]: + p.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') + p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions') + p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders') + p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders') + p.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.') + p.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') + p.add_argument('--skip-extension', '-E', type=str, action='append', help='File Extension to skip.') + p.add_argument('--skip-folder', '-O', type=str, action='append', help='Folder to skip.') + p.add_argument('--skip-size', '-Z', type=int, default=0, + help='Minimum file size to consider for fingerprinting (optional - default 0 bytes [unlimited])') + # Global Scan/GRPC options for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: p.add_argument('--key', '-k', type=str, @@ -374,8 +375,9 @@ def wfp(parser, args): scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value # Skip snippet generation or not scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, obfuscate=args.obfuscate, scan_options=scan_options, all_extensions=args.all_extensions, - all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm) - + all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm, + skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder + ) if args.stdin: contents = sys.stdin.buffer.read() scanner.wfp_contents(args.stdin, contents, scan_output) @@ -530,14 +532,15 @@ def scan(parser, args): scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, grpc_url=args.api2url, obfuscate=args.obfuscate, ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy, - pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, hpsm=args.hpsm + pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, hpsm=args.hpsm, + skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder ) if args.wfp: if not scanner.is_file_or_snippet_scan(): print_stderr(f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})') exit(1) if scanner.is_dependency_scan() and not args.dep: - print_stderr(f'Error: Cannot specify WFP & Dependency scanning without a dependency file ({--dep})') + print_stderr(f'Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)') exit(1) scanner.scan_wfp_with_options(args.wfp, args.dep) elif args.stdin: diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index b3e8cd1c..59abd066 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -58,7 +58,7 @@ FILTERED_DIR_EXT = { # Folder endings to skip ".egg-info" } -FILTERED_EXT = { # File extensions to skip +FILTERED_EXT = [ # File extensions to skip ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake", ".cnf", ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css", @@ -78,7 +78,7 @@ # File endings "-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3" -} +] FILTERED_FILES = { # Files to skip "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile" @@ -100,12 +100,17 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, - ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False + ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False, + skip_size: int = 0, skip_extensions=None, skip_folders=None ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning """ super().__init__(debug, trace, quiet) + if skip_folders is None: + skip_folders = [] + if skip_extensions is None: + skip_extensions = [] self.wfp = wfp if wfp else "scanner_output.wfp" self.scan_output = scan_output self.output_format = output_format @@ -117,6 +122,8 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.scan_options = scan_options self._skip_snippets = True if not scan_options & ScanType.SCAN_SNIPPETS.value else False self.hpsm = hpsm + self.skip_folders = skip_folders + self.skip_size = skip_size ver_details = Scanner.version_details() self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, @@ -143,6 +150,9 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str self.post_file_count = post_size if post_size > 0 else 32 # Max number of files for any given POST (default 32) if self._skip_snippets: self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets + self.skip_extensions = FILTERED_EXT + if skip_extensions: # Append extra file extensions to skip + self.skip_extensions.extend(skip_extensions) def __filter_files(self, files: list) -> list: """ @@ -160,8 +170,8 @@ def __filter_files(self, files: list) -> list: if f_lower in FILTERED_FILES: # Check for exact files to ignore ignore = True if not ignore: - for ending in FILTERED_EXT: # Check for file endings to ignore - if f_lower.endswith(ending): + for ending in self.skip_extensions: # Check for file endings to ignore (static and user supplied) + if ending and f_lower.endswith(ending): ignore = True break if not ignore: @@ -181,10 +191,12 @@ def __filter_dirs(self, dirs: list) -> list: ignore = True if not ignore and not self.all_folders: # Skip this check if we're allowing all folders d_lower = d.lower() - if d_lower in FILTERED_DIRS: # Ignore specific folders + if d_lower in FILTERED_DIRS: # Ignore specific folders (case insensitive) + ignore = True + elif self.skip_folders and d in self.skip_folders: # Ignore user-supplied folders (case sensitive) ignore = True if not ignore: - for de in FILTERED_DIR_EXT: # Ignore specific folder endings + for de in FILTERED_DIR_EXT: # Ignore specific folder endings (case insensitive) if d_lower.endswith(de): ignore = True break @@ -385,7 +397,8 @@ def scan_folder(self, scan_dir: str) -> bool: except Exception as e: self.print_trace( f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink - if f_size > 0: # Ignore broken links and empty files + # Ignore broken links and empty files or if a user-specified size limit is supplied + if f_size > 0 and (self.skip_size <= 0 or f_size > self.skip_size): self.print_trace(f'Fingerprinting {path}...') if spinner: spinner.next() @@ -598,7 +611,7 @@ def scan_file(self, file: str) -> bool: success = False return success - def scan_files(self, files: list[str]) -> bool: + def scan_files(self, files: []) -> bool: """ Scan the specified list of files, producing fingerprints, send to the SCANOSS API and return results Please note that by providing an explicit list you bypass any exclusions that may be defined on the scanner @@ -637,7 +650,7 @@ def scan_files(self, files: list[str]) -> bool: spinner.next() wfp = self.winnowing.wfp_for_file(file, file) if wfp is None or wfp == '': - self.print_stderr(f'Warning: No WFP returned for {path}') + self.print_stderr(f'Warning: No WFP returned for {file}') continue if save_wfps_for_print: wfp_list.append(wfp) @@ -681,12 +694,12 @@ def scan_files(self, files: list[str]) -> bool: if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: - Scanner.print_stderr(f'Warning: No files found to scan in folder: {scan_dir}') + Scanner.print_stderr(f'Warning: No files found to scan from: {files}') return success - def scan_files_with_options(self, files: list[str], deps_file: str = None, file_map: dict = None) -> bool: + def scan_files_with_options(self, files: [], deps_file: str = None, file_map: dict = None) -> bool: """ - Scan the given folder for whatever scaning options that have been configured + Scan the given list of files for whatever scaning options that have been configured :param files: list of files to scan :param deps_file: pre-parsed dependency file to decorate :param file_map: mapping of obfuscated files back into originals @@ -697,7 +710,7 @@ def scan_files_with_options(self, files: list[str], deps_file: str = None, file_ raise Exception(f"ERROR: Please specify a list of files to scan") if not self.is_file_or_snippet_scan(): raise Exception(f"ERROR: file or snippet scan options have to be set to scan files: {files}") - if self.is_dependency_scan(): + if self.is_dependency_scan() or deps_file: raise Exception(f"ERROR: The dependency scan option is currently not supported when scanning a list of files") if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') @@ -852,7 +865,7 @@ def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): - raise Exception(f"ERROR: No scan options defined to scan folder: {scan_dir}") + raise Exception(f"ERROR: No scan options defined to scan WFP: {wfp}") if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') @@ -860,18 +873,17 @@ def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) if not self.threaded_deps.run(deps_file=deps_file, wait=False): # Kick off a background dependency scan success = False if self.is_file_or_snippet_scan(): - if not self.scan_wfp_file_threaded(wfp_file, file_map): + if not self.scan_wfp_file_threaded(wfp_file): success = False if self.threaded_scan: if not self.__finish_scan_threaded(file_map): success = False return success - def scan_wfp_file_threaded(self, file: str = None, file_map: dict = None) -> bool: + def scan_wfp_file_threaded(self, file: str = None) -> bool: """ Scan the contents of the specified WFP file (threaded) :param file: WFP file to scan (optional) - :param file_map: mapping of obfuscated files back into originals (optional) return: True if successful, False otherwise """ success = True diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index d0251cc4..e5d22e78 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -43,7 +43,8 @@ from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse -from .api.components.v2.scanoss_components_pb2 import CompSearchRequest, CompSearchResponse, CompVersionRequest, CompVersionResponse +from .api.components.v2.scanoss_components_pb2 import (CompSearchRequest, CompSearchResponse, + CompVersionRequest, CompVersionResponse) from .scanossbase import ScanossBase from . import __version__ diff --git a/src/scanoss/threadeddependencies.py b/src/scanoss/threadeddependencies.py index 43463eb6..289fb6a9 100644 --- a/src/scanoss/threadeddependencies.py +++ b/src/scanoss/threadeddependencies.py @@ -112,8 +112,8 @@ def scan_dependencies(self) -> None: if deps is None: self.print_stderr(f'Problem searching for dependencies for: {what_to_scan}') self._errors = True - elif not deps: - self.print_trace(f'No dependencies found to decorate for: {what_to_scan}') + elif not deps or len(deps.get("files", [])) == 0: + self.print_debug(f'No dependencies found to decorate for: {what_to_scan}') else: decorated_deps = self.grpc_api.get_dependencies(deps) if decorated_deps: From 91c0f1af58e15b272014a1632d8e8611c1fd4d5f Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:11:43 -0300 Subject: [PATCH 126/489] CLIS-102 Installs command line JSON processor jq (#31) --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 95dc4617..3e5449b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,11 @@ COPY --from=builder /root/.local /root/.local ENV PATH=/root/.local/bin:$PATH ENV GRPC_POLL_STRATEGY=poll +RUN apt-get update \ + && apt-get install -y --no-install-recommends jq \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + VOLUME /scanoss WORKDIR /scanoss From 9a0f1689ee351a412b3309d7a11956ced1467c30 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:40:44 -0400 Subject: [PATCH 127/489] MD5 and Snippet Filtering (#32) * do not report error when no dependency files found * added scan and wfp filter options * added WFP HPSM test * init cut at filtering files on MD5, HPSM and Snippet ID strings * modify HPSM sprip * definitive implementation of HPSM striping * extract filtering to functions * add wfp and hpsm strip test * file and snippet filtering * update message debug * fix snippet cmd option * hpsm/strip warning --------- Co-authored-by: core software devel --- CHANGELOG.md | 8 ++++ CLIENT_HELP.md | 13 ++++++- setup.cfg | 2 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 13 ++++++- src/scanoss/scanner.py | 15 +++++--- src/scanoss/winnowing.py | 83 ++++++++++++++++++++++++++++++++++++++-- tests/winnowing-test.py | 24 ++++++++++++ 8 files changed, 145 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eed0892e..33ff47b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.11.0] - 2024-03-13 +### Added +- Added scan/wfp file filtering options + - Exclude files matching MD5 `--skip-md5` (repeat as needed) + - Strip code fragments using HPSM `--strip-hpsm` (repeat as needed) + - Strip code fragments using snippet IDs `--strip-snippet` (repeat as needed) + ## [1.10.0] - 2024-02-09 ### Added - Added scan/wfp file filtering options @@ -291,3 +298,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.8.0]: https://github.com/scanoss/scanoss.py/compare/v1.7.0...v1.8.0 [1.9.0]: https://github.com/scanoss/scanoss.py/compare/v1.8.0...v1.9.0 [1.10.0]: https://github.com/scanoss/scanoss.py/compare/v1.9.0...v1.10.0 +[1.11.0]: https://github.com/scanoss/scanoss.py/compare/v1.10.0...v1.11.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index e0131616..5772819b 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -184,11 +184,22 @@ scanoss-py scan -o scan-results.json src ``` ### Scan a project folder with dependencies -The following command scans the `src` folder file, snippet & dependency matches, writing the output to `scan-results.json`: +The following command scans the `src` folder for file, snippet & dependency matches, writing the output to `scan-results.json`: ```bash scanoss-py scan -o scan-results.json -D src ``` +### Scan a project folder skipping files and snippets +The following command scans the `src` folder writing the output to `scan-results.json` skipping the following: +- MD5 file `37f7cd1e657aa3c30ece35995b4c59e5` +- Header files `.h` +- Files smaller than 512 byes +- Files inside folder `internal` +- Snippets matching `d5e54c33,b03faabe` +```bash +scanoss-py scan -o scan-results.json -5 37f7cd1e657aa3c30ece35995b4c59e5 -E '.h' -Z 512 -O internal -N 'd5e54c33,b03faabe' src +``` + ### Converting RAW results into other formats The following command provides the capability to convert the RAW scan results from a SCANOSS scan into multiple different formats, including CycloneDX, SPDX Lite, CSV, etc. For the full set of formats, please run: diff --git a/setup.cfg b/setup.cfg index e84568a2..261d39df 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ install_requires = [options.extras_require] fast_winnowing = - scanoss_winnowing>=0.3.0 + scanoss_winnowing>=0.5.0 [options.packages.find] where = src diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 151bffdf..35cf6bb9 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.10.0' +__version__ = '1.11.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 9916c02d..cf495628 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -260,6 +260,9 @@ def setup_args() -> None: p.add_argument('--skip-folder', '-O', type=str, action='append', help='Folder to skip.') p.add_argument('--skip-size', '-Z', type=int, default=0, help='Minimum file size to consider for fingerprinting (optional - default 0 bytes [unlimited])') + p.add_argument('--skip-md5', '-5', type=str, action='append', help='Skip files matching MD5.') + p.add_argument('--strip-hpsm', '-G', type=str, action='append', help='Strip HPSM string from WFP.') + p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.') # Global Scan/GRPC options for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: @@ -367,6 +370,8 @@ def wfp(parser, args): print_stderr('Please specify a file/folder or STDIN (--stdin)') parser.parse_args([args.subparser, '-h']) exit(1) + if args.strip_hpsm and not args.hpsm and not args.quiet: + print_stderr(f'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') scan_output: str = None if args.output: scan_output = args.output @@ -376,7 +381,8 @@ def wfp(parser, args): scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, obfuscate=args.obfuscate, scan_options=scan_options, all_extensions=args.all_extensions, all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm, - skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder + skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder, + skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet ) if args.stdin: contents = sys.stdin.buffer.read() @@ -473,6 +479,8 @@ def scan(parser, args): exit(1) if not Scanner.valid_json_file(args.dep): # Make sure it's a valid JSON file exit(1) + if args.strip_hpsm and not args.hpsm and not args.quiet: + print_stderr(f'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') scan_output: str = None if args.output: @@ -533,7 +541,8 @@ def scan(parser, args): grpc_url=args.api2url, obfuscate=args.obfuscate, ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, hpsm=args.hpsm, - skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder + skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder, + skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet ) if args.wfp: if not scanner.is_file_or_snippet_scan(): diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 59abd066..3db58e13 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -101,7 +101,8 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False, - skip_size: int = 0, skip_extensions=None, skip_folders=None + skip_size: int = 0, skip_extensions=None, skip_folders=None, + strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None ): """ Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning @@ -127,7 +128,9 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str ver_details = Scanner.version_details() self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, - all_extensions=all_extensions, obfuscate=obfuscate, hpsm=self.hpsm + all_extensions=all_extensions, obfuscate=obfuscate, hpsm=self.hpsm, + strip_hpsm_ids=strip_hpsm_ids, strip_snippet_ids=strip_snippet_ids, + skip_md5_ids=skip_md5_ids ) self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, @@ -404,7 +407,7 @@ def scan_folder(self, scan_dir: str) -> bool: spinner.next() wfp = self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) if wfp is None or wfp == '': - self.print_stderr(f'Warning: No WFP returned for {path}') + self.print_debug(f'No WFP returned for {path}. Skipping.') continue if save_wfps_for_print: wfp_list.append(wfp) @@ -650,10 +653,10 @@ def scan_files(self, files: []) -> bool: spinner.next() wfp = self.winnowing.wfp_for_file(file, file) if wfp is None or wfp == '': - self.print_stderr(f'Warning: No WFP returned for {file}') - continue + self.print_debug(f'No WFP returned for {file}. Skipping.') + continue if save_wfps_for_print: - wfp_list.append(wfp) + wfp_list.append(wfp) file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode("utf-8")) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 47ca74fb..1b9da304 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -29,6 +29,7 @@ """ import hashlib import pathlib +import re from crc32c import crc32c from binaryornot.check import is_binary @@ -57,7 +58,7 @@ ".o", ".a", ".so", ".obj", ".dll", ".lib", ".out", ".app", ".bin", ".lst", ".dat", ".json", ".htm", ".html", ".xml", ".md", ".txt", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".pages", ".key", ".numbers", - ".pdf", ".min.js", ".mf", ".sum", ".woff", ".woff2" + ".pdf", ".min.js", ".mf", ".sum", ".woff", ".woff2", '.xsd', ".pom" } CRC8_MAXIM_DOW_TABLE_SIZE = 0x100 @@ -111,7 +112,8 @@ class Winnowing(ScanossBase): def __init__(self, size_limit: bool = False, debug: bool = False, trace: bool = False, quiet: bool = False, skip_snippets: bool = False, post_size: int = 32, all_extensions: bool = False, - obfuscate: bool = False, hpsm: bool = False + obfuscate: bool = False, hpsm: bool = False, + strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None ): """ Instantiate Winnowing class @@ -121,6 +123,12 @@ def __init__(self, size_limit: bool = False, debug: bool = False, trace: bool = Limit the size of a fingerprint to 32k (post size) - Default False """ super().__init__(debug, trace, quiet) + if strip_hpsm_ids is None: + strip_hpsm_ids = [] + if strip_snippet_ids is None: + strip_snippet_ids = [] + if skip_md5_ids is None: + skip_md5_ids = [] self.size_limit = size_limit self.skip_snippets = skip_snippets self.max_post_size = post_size * 1024 if post_size > 0 else MAX_POST_SIZE @@ -128,6 +136,9 @@ def __init__(self, size_limit: bool = False, debug: bool = False, trace: bool = self.obfuscate = obfuscate self.ob_count = 1 self.file_map = {} if obfuscate else None + self.skip_md5_ids = skip_md5_ids + self.strip_hpsm_ids = strip_hpsm_ids + self.strip_snippet_ids = strip_snippet_ids self.hpsm = hpsm if hpsm: self.crc8_maxim_dow_table = [] @@ -221,6 +232,63 @@ def is_binary(self, path: str): return binary_path return False + def __strip_hpsm(self, file: str, hpsm: str) -> str: + """ + Strip off request HPSM IDs if requested + + :param file: name of the fingerprinted file + :param hpsm: HPSM string + :return: modified HPSM (if necessary) + """ + hpsm_len = len(hpsm) + if self.strip_hpsm_ids and hpsm_len > 1: + # Check for HPSM ID strings to remove. The size of the sequence must be conserved. + for hpsm_id in self.strip_hpsm_ids: + hpsm_id_index = hpsm.find(hpsm_id) + hpsm_id_len = len(hpsm_id) + # If the position is odd, we need to overwrite one byte before. + if hpsm_id_index % 2 == 1: + hpsm_id_index = hpsm_id_index - 1 + # If the size of the sequence is even, we need to overwrite one byte after. + if hpsm_id_len % 2 == 0: + hpsm_id_len = hpsm_id_len + 1 + hpsm_id_len = hpsm_id_len + 1 + # If the position is even and the size is odd, we need to overwrite one byte after. + elif hpsm_id_len % 2 == 1: + hpsm_id_len = hpsm_id_len + 1 + + to_remove = hpsm[hpsm_id_index:hpsm_id_index + hpsm_id_len] + self.print_debug(f'HPSM ID {to_remove} to replace') + # Calculate the XOR of each byte to produce the correct ignore sequence. + replacement = ''.join( + [format(int(to_remove[i:i + 2], 16) ^ 0xFF, '02x') for i in range(0, len(to_remove), 2)]) + + self.print_debug(f'HPSM ID replacement {replacement}') + # Overwrite HPSM bytes to be removed. + hpsm = hpsm.replace(to_remove, replacement) + if hpsm_len != len(hpsm): + self.print_stderr(f'wrong HPSM values from {file}') + return hpsm + + def __strip_snippets(self, file: str, wfp: str) -> str: + """ + Strip snippet IDs from the WFP + + :param file: name of fingerprinted file + :param wfp: WFP to clean + :return: updated WFP + """ + wfp_len = len(wfp) + for snippet_id in self.strip_snippet_ids: # Remove exact snippet strings + wfp = wfp.replace(snippet_id, '') + if wfp_len > len(wfp): + wfp = re.sub(r'(,)\1+', ',', wfp) # Remove multiple 'empty comma' blocks + wfp = wfp.replace(',\n', '\n') # Remove trailing comma + wfp = wfp.replace('=,', '=') # Remove leading comma + wfp = re.sub(r'\d+=\s+', '', wfp) # Cleanup empty lines + self.print_debug(f'Stripped snippet ids from {file}') + return wfp + def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: """ Generate a Winnowing fingerprint (WFP) for the given file contents @@ -234,6 +302,9 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: WFP string """ file_md5 = hashlib.md5(contents).hexdigest() + if self.skip_md5_ids and file_md5 in self.skip_md5_ids: + self.print_debug(f'Skipping MD5 file name for {file_md5}: {file}') + return '' # Print file line content_length = len(contents) wfp_filename = file @@ -248,8 +319,9 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: return wfp # Add HPSM if self.hpsm: - hpsm = self.calc_hpsm(contents) - wfp += 'hpsm={0}\n'.format(hpsm) + hpsm = self.__strip_hpsm(file, self.calc_hpsm(contents)) + if len(hpsm) > 0: + wfp += f'hpsm={hpsm}\n' # Initialize variables gram = '' window = [] @@ -309,6 +381,9 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: if wfp is None or wfp == '': self.print_stderr(f'Warning: No WFP content data for {file}') + elif self.strip_snippet_ids: + wfp = self.__strip_snippets(file, wfp) + return wfp def calc_hpsm(self, content): diff --git a/tests/winnowing-test.py b/tests/winnowing-test.py index 56dc4f4e..52e05e43 100644 --- a/tests/winnowing-test.py +++ b/tests/winnowing-test.py @@ -51,6 +51,30 @@ def test_snippet_skip(self): wfp = winnowing.wfp_for_contents(filename, False, content_types) print(f'WFP for {filename}: {wfp}') self.assertIsNotNone(wfp) + + def test_snippet_strip(self): + winnowing = Winnowing(debug=True, hpsm=True, + strip_snippet_ids=['d5e54c33,b03faabe'], + strip_hpsm_ids=['0d2fffaffc62d18']) + filename = "test-file.py" + with open(__file__, 'rb') as f: + contents = f.read() + print('--- Test snippet and HPSM strip ---') + wfp = winnowing.wfp_for_contents(filename, False, contents) + found = 0 + print(f'WFP for {filename}: {wfp}') + try: + found = wfp.index('d5e54c33,b03faabe') + except ValueError: + found = -1 + self.assertEqual(found, -1) + + try: + found = wfp.index('0d2fffaffc62d18') + except ValueError: + found = -1 + self.assertEqual(found, -1) + if __name__ == '__main__': From 71eee1f68a0cfae91d2dbe3c0793de412b656f9d Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:48:09 -0300 Subject: [PATCH 128/489] CLI-104 Installs curl on Docker file (#33) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3e5449b0..7280b019 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ ENV PATH=/root/.local/bin:$PATH ENV GRPC_POLL_STRATEGY=poll RUN apt-get update \ - && apt-get install -y --no-install-recommends jq \ + && apt-get install -y --no-install-recommends jq curl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From ce4262f66d4f45240451e4d99c78b4423735f035 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Mon, 18 Mar 2024 11:25:39 -0300 Subject: [PATCH 129/489] CLI-105 Updates CHANGELOG.md file (#34) --- CHANGELOG.md | 6 ++++++ src/scanoss/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33ff47b4..57165e4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.11.1] - 2024-03-18 +### Added +- Integrate CURL and jq + - Includes CURL and jq within the Docker image to facilitate seamless interactions with third-party integrations. + ## [1.11.0] - 2024-03-13 ### Added - Added scan/wfp file filtering options @@ -299,3 +304,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.9.0]: https://github.com/scanoss/scanoss.py/compare/v1.8.0...v1.9.0 [1.10.0]: https://github.com/scanoss/scanoss.py/compare/v1.9.0...v1.10.0 [1.11.0]: https://github.com/scanoss/scanoss.py/compare/v1.10.0...v1.11.0 +[1.11.1]: https://github.com/scanoss/scanoss.py/compare/v1.11.0...v1.11.1 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 35cf6bb9..437a3113 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.11.0' +__version__ = '1.11.1' From 15bcfce6628d0f816b0b2dabbd94ed35c8606815 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:44:36 -0400 Subject: [PATCH 130/489] Update default API URLs (#35) * update free and premium default urls --- .github/workflows/container-local-test.yml | 2 +- CHANGELOG.md | 8 +++++++- CLIENT_HELP.md | 6 +++--- PACKAGE.md | 2 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 8 ++++---- src/scanoss/scanossapi.py | 6 +++--- src/scanoss/scanossgrpc.py | 4 ++-- 8 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index ecb61c49..1e20ae8f 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -1,5 +1,5 @@ name: Build/Test Local Container -# Build a docker image on demand and run a local test (connecting to osskb.org) +# Build a docker image on demand and run a local test (connecting to api.osskb.org) on: workflow_dispatch: diff --git a/CHANGELOG.md b/CHANGELOG.md index 57165e4d..c77bd454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.12.0] - 2024-03-26 +### Changed +- Updated free default URL to now point to `https://api.osskb.org` +- Updated premium default URL to now point to `https://api.scanoss.com` + ## [1.11.1] - 2024-03-18 ### Added - Integrate CURL and jq @@ -304,4 +309,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.9.0]: https://github.com/scanoss/scanoss.py/compare/v1.8.0...v1.9.0 [1.10.0]: https://github.com/scanoss/scanoss.py/compare/v1.9.0...v1.10.0 [1.11.0]: https://github.com/scanoss/scanoss.py/compare/v1.10.0...v1.11.0 -[1.11.1]: https://github.com/scanoss/scanoss.py/compare/v1.11.0...v1.11.1 \ No newline at end of file +[1.11.1]: https://github.com/scanoss/scanoss.py/compare/v1.11.0...v1.11.1 +[1.11.1]: https://github.com/scanoss/scanoss.py/compare/v1.11.1...v1.12.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 5772819b..3adf283b 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -113,11 +113,11 @@ The `scanoss-py` CLI provides a utility command to help identify if traffic to t Simply run the following commands find out: * auto - * `scanoss-py utils pac-proxy --pac auto --url https://osskb.org` + * `scanoss-py utils pac-proxy --pac auto --url https://api.osskb.org` * file - * `scanoss-py utils pac-proxy --pac file://proxy.pac --url https://osskb.org` + * `scanoss-py utils pac-proxy --pac file://proxy.pac --url https://api.osskb.org` * url - * `scanoss-py utils pac-proxy --pac https://path.to/proxy.pac --url https://osskb.org` + * `scanoss-py utils pac-proxy --pac https://path.to/proxy.pac --url https://api.osskb.org` ## GRPCIO Library installation for Apple Silicon (before 1.5.3) Versions of [grpcio](https://pypi.org/project/grpcio) prior to `1.5.3` did not contain a binary wheel for Apple Silicon. diff --git a/PACKAGE.md b/PACKAGE.md index 9848cf17..7e4f8b26 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -117,7 +117,7 @@ if __name__ == "__main__": ``` ## Scanning URL and API Key -By Default, scanoss uses the API URL endpoint for SCANOSS OSS KB: https://osskb.org/api/scan/direct. +By Default, scanoss uses the API URL endpoint for SCANOSS OSS KB: https://api.osskb.org/scan/direct. This API does not require an API key. These values can be changed from the command line using: diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 437a3113..1912d2aa 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.11.1' +__version__ = '1.12.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index cf495628..00797930 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -239,13 +239,13 @@ def setup_args() -> None: p_p_proxy.add_argument('--pac', required=False, type=str, default="auto", help='Proxy auto configuration. Specify a file, http url or "auto" to try to discover it.' ) - p_p_proxy.add_argument('--url', required=False, type=str, default="https://osskb.org/api", - help='URL to test (default: https://osskb.org/api).') + p_p_proxy.add_argument('--url', required=False, type=str, default="https://api.osskb.org", + help='URL to test (default: https://api.osskb.org).') # Global Scan command options for p in [p_scan]: p.add_argument('--apiurl', type=str, - help='SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)') + help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)') p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') # Global Scan/Fingerprint filter options @@ -281,7 +281,7 @@ def setup_args() -> None: # Global GRPC options for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: p.add_argument('--api2url', type=str, - help='SCANOSS gRPC API 2.0 URL (optional - default: https://osskb.org)') + help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)') p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' 'Can also use the environment variable "grcp_proxy=:"') diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index ad68fb63..02b151af 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -38,8 +38,8 @@ from . import __version__ -DEFAULT_URL = "https://osskb.org/api/scan/direct" # default free service URL -DEFAULT_URL2 = "https://scanoss.com/api/scan/direct" # default premium service URL +DEFAULT_URL = "https://api.osskb.org/scan/direct" # default free service URL +DEFAULT_URL2 = "https://api.scanoss.com/scan/direct" # default premium service URL SCANOSS_SCAN_URL = os.environ.get("SCANOSS_SCAN_URL") if os.environ.get("SCANOSS_SCAN_URL") else DEFAULT_URL SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' @@ -60,7 +60,7 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st :param sbom_path: Input SBOM file to match scan type (default None) :param scan_format: Scan format (default plain) :param flags: Scanning flags (default None) - :param url: API URL (default https://osskb.org/api/scan/direct) + :param url: API URL (default https://api.osskb.org/scan/direct) :param api_key: API Key (default None) :param debug: Enable debug (default False) :param trace: Enable trace (default False) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index e5d22e78..80bec813 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -48,8 +48,8 @@ from .scanossbase import ScanossBase from . import __version__ -DEFAULT_URL = "https://osskb.org" # default free service URL -DEFAULT_URL2 = "https://scanoss.com" # default premium service URL +DEFAULT_URL = "https://api.osskb.org" # default free service URL +DEFAULT_URL2 = "https://api.scanoss.com" # default premium service URL SCANOSS_GRPC_URL = os.environ.get("SCANOSS_GRPC_URL") if os.environ.get("SCANOSS_GRPC_URL") else DEFAULT_URL SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' From b7d1abec12abeeeeb99d9606d7bdfa36173b6cf7 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Fri, 12 Apr 2024 11:32:53 -0300 Subject: [PATCH 131/489] SP-534 Removes '.whl' from filtered extensions --- CHANGELOG.md | 7 ++++++- src/scanoss/__init__.py | 2 +- src/scanoss/scanner.py | 2 +- src/scanoss/winnowing.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c77bd454..9ea41991 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.12.1] - 2024-04-12 +### Changed +- Removed '.whl' file extension from filtered extensions + ## [1.12.0] - 2024-03-26 ### Changed - Updated free default URL to now point to `https://api.osskb.org` @@ -310,4 +314,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.10.0]: https://github.com/scanoss/scanoss.py/compare/v1.9.0...v1.10.0 [1.11.0]: https://github.com/scanoss/scanoss.py/compare/v1.10.0...v1.11.0 [1.11.1]: https://github.com/scanoss/scanoss.py/compare/v1.11.0...v1.11.1 -[1.11.1]: https://github.com/scanoss/scanoss.py/compare/v1.11.1...v1.12.0 +[1.12.0]: https://github.com/scanoss/scanoss.py/compare/v1.11.1...v1.12.0 +[1.12.1]: https://github.com/scanoss/scanoss.py/compare/v1.12.0...v1.12.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 1912d2aa..b8a61e83 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.12.0' +__version__ = '1.12.1' diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 3db58e13..674fce9f 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -71,7 +71,7 @@ ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", - ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".woff2", ".xht", + ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".woff", ".woff2", ".xht", ".xhtml", ".xls", ".xlsx", ".xml", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 1b9da304..985dd858 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -58,7 +58,7 @@ ".o", ".a", ".so", ".obj", ".dll", ".lib", ".out", ".app", ".bin", ".lst", ".dat", ".json", ".htm", ".html", ".xml", ".md", ".txt", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".pages", ".key", ".numbers", - ".pdf", ".min.js", ".mf", ".sum", ".woff", ".woff2", '.xsd', ".pom" + ".pdf", ".min.js", ".mf", ".sum", ".woff", ".woff2", '.xsd', ".pom", ".whl", } CRC8_MAXIM_DOW_TABLE_SIZE = 0x100 From af69757ed882e2ad3bf1edc76bdd0819c1b5c95b Mon Sep 17 00:00:00 2001 From: eeisegn Date: Mon, 15 Apr 2024 17:40:29 +0100 Subject: [PATCH 132/489] auto determine version tag --- .github/workflows/version-tag.yml | 35 +++++++++++++++++++ tools/get_next_version.sh | 56 +++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 .github/workflows/version-tag.yml create mode 100755 tools/get_next_version.sh diff --git a/.github/workflows/version-tag.yml b/.github/workflows/version-tag.yml new file mode 100644 index 00000000..d86ad75f --- /dev/null +++ b/.github/workflows/version-tag.yml @@ -0,0 +1,35 @@ +name: Repo Version Tagging +# This workflow will read the version details from the repo and apply a branch + +on: + workflow_dispatch: + inputs: + run_for_real: + required: true + default: false + type: boolean + description: "Apply next tag (or Dry Run)" + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10.x' + - name: Determine Tag + id: taggerVersion + run: | + app_version=$(tools/get_next_version.sh) + echo "tag_ver=$app_version" >> $GITHUB_OUTPUT + + - name: Run Tagging + if: ${{ inputs.run_for_real }} + id: taggerApply + run: | + echo "Applying tag $app_version ..." + diff --git a/tools/get_next_version.sh b/tools/get_next_version.sh new file mode 100755 index 00000000..7cf4d2f1 --- /dev/null +++ b/tools/get_next_version.sh @@ -0,0 +1,56 @@ +#!/bin/bash +### +# SPDX-License-Identifier: MIT +# +# Copyright (c) 2024, SCANOSS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +### +# +# Determine the latest tag associated with this repo and echo to stdout +# +export d=`dirname "$0"` + +if [ "$d" = "" ] ; then + export d=. +fi + +version=$(git describe --tags --abbrev=0) +if [[ -z "$version" ]] ; then + version=$(git describe --tags "$(git rev-list --tags --max-count=1)") +fi +if [[ -z "$version" ]] ; then + echo "Error: Failed to determine a valid version number" >&2 + exit 1 +fi +python_version=$($d/../version.py) +if [ $? -eq 1 ] || [[ "$python_version" = "" ]]; then + echo "Error: failed to get python app version." + exit 1 +fi +semver_python="v$python_version" + +echo "Latest Tag: $version, Python: $python_version" >&2 + +if [[ "$version" == "$semver_python" ]] ; then + echo "Latest tag and python version are the same: $version" >&2 + exit 1 +fi +echo "$semver_python" +exit 0 From 2359618aaf7868dccc74b6574a5cabb925bc24ed Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:06:33 +0200 Subject: [PATCH 133/489] SP-548 version tagging (#37) * Add release tagging workflow --- .github/workflows/version-tag.yml | 18 +++++++++++------- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- tools/get_next_version.sh | 11 +++++++---- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/version-tag.yml b/.github/workflows/version-tag.yml index d86ad75f..0d3ee83a 100644 --- a/.github/workflows/version-tag.yml +++ b/.github/workflows/version-tag.yml @@ -11,25 +11,29 @@ on: description: "Apply next tag (or Dry Run)" jobs: - deploy: + version-tagging: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: '0' - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' - name: Determine Tag id: taggerVersion run: | app_version=$(tools/get_next_version.sh) - echo "tag_ver=$app_version" >> $GITHUB_OUTPUT + echo "New Proposed tag: $app_version" + echo "package_app_version=$app_version" >> $GITHUB_ENV - - name: Run Tagging + - name: Apply Tag if: ${{ inputs.run_for_real }} id: taggerApply run: | - echo "Applying tag $app_version ..." - + echo "Applying tag ${{env.package_app_version}} ..." + git tag "${{env.package_app_version}}" + echo "Pushing changes..." + git push --tags + diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea41991..97888303 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.12.2] - 2024-04-15 +### Added +- Added [tagging workflow](.github/workflows/version-tag.yml) to aid release generation + ## [1.12.1] - 2024-04-12 ### Changed - Removed '.whl' file extension from filtered extensions @@ -316,3 +320,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.11.1]: https://github.com/scanoss/scanoss.py/compare/v1.11.0...v1.11.1 [1.12.0]: https://github.com/scanoss/scanoss.py/compare/v1.11.1...v1.12.0 [1.12.1]: https://github.com/scanoss/scanoss.py/compare/v1.12.0...v1.12.1 +[1.12.2]: https://github.com/scanoss/scanoss.py/compare/v1.12.1...v1.12.2 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b8a61e83..eab5d8c7 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.12.1' +__version__ = '1.12.2' diff --git a/tools/get_next_version.sh b/tools/get_next_version.sh index 7cf4d2f1..d0d299f0 100755 --- a/tools/get_next_version.sh +++ b/tools/get_next_version.sh @@ -23,14 +23,14 @@ # THE SOFTWARE. ### # -# Determine the latest tag associated with this repo and echo to stdout +# Get the defined package version and compare to the latest tag. Echo the new tag if it doesn't already exist. # -export d=`dirname "$0"` - +export d=$(dirname "$0") if [ "$d" = "" ] ; then export d=. fi +# Get latest git tagged version version=$(git describe --tags --abbrev=0) if [[ -z "$version" ]] ; then version=$(git describe --tags "$(git rev-list --tags --max-count=1)") @@ -39,15 +39,18 @@ if [[ -z "$version" ]] ; then echo "Error: Failed to determine a valid version number" >&2 exit 1 fi +# Get Python package version python_version=$($d/../version.py) if [ $? -eq 1 ] || [[ "$python_version" = "" ]]; then echo "Error: failed to get python app version." exit 1 fi +# Convert to semver (with 'v' prefix) semver_python="v$python_version" -echo "Latest Tag: $version, Python: $python_version" >&2 +echo "Latest Tag: $version, Python Version: $python_version" >&2 +# If the two versions are the same abort, as we don't want to apply the same tag again if [[ "$version" == "$semver_python" ]] ; then echo "Latest tag and python version are the same: $version" >&2 exit 1 From bb7dbfcd93c2f6fd2f31e9d861c687a2a989c813 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:25:57 +0200 Subject: [PATCH 134/489] SP-548 Added Custom GH Token (#38) * Added custom token for tagging --- .github/workflows/version-tag.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/version-tag.yml b/.github/workflows/version-tag.yml index 0d3ee83a..c327eeda 100644 --- a/.github/workflows/version-tag.yml +++ b/.github/workflows/version-tag.yml @@ -10,11 +10,15 @@ on: type: boolean description: "Apply next tag (or Dry Run)" +concurrency: production + jobs: version-tagging: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + env: + GITHUB_TOKEN: ${{ secrets.SC_GH_TAG_TOKEN }} with: fetch-depth: '0' - name: Set up Python From 5988ff4ed63c979d676d5a7482ce71217bf59f26 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Tue, 14 May 2024 13:49:26 +0200 Subject: [PATCH 135/489] Fixed license missing issue during export (#39) --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 18 ++++++++++-------- src/scanoss/spdxlite.py | 26 ++++++++++++++------------ 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97888303..224e69d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.12.3] - 2024-05-13 +### Fixed +- Fixed export issue when license details are missing (SPDX/CycloneDX) + ## [1.12.2] - 2024-04-15 ### Added - Added [tagging workflow](.github/workflows/version-tag.yml) to aid release generation @@ -321,3 +325,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.12.0]: https://github.com/scanoss/scanoss.py/compare/v1.11.1...v1.12.0 [1.12.1]: https://github.com/scanoss/scanoss.py/compare/v1.12.0...v1.12.1 [1.12.2]: https://github.com/scanoss/scanoss.py/compare/v1.12.1...v1.12.2 +[1.12.3]: https://github.com/scanoss/scanoss.py/compare/v1.12.2...v1.12.3 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index eab5d8c7..499996e5 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.12.2' +__version__ = '1.12.3' diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 1e267bed..75df57ac 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -83,12 +83,13 @@ def parse(self, data: json): fd[field] = deps.get(field, '') licenses = deps.get('licenses') fdl = [] - dc = [] - for lic in licenses: - name = lic.get("name") - if name not in dc: # Only save the license name once - fdl.append({'id': name}) - dc.append(name) + if licenses: + dc = [] + for lic in licenses: + name = lic.get("name") + if name not in dc: # Only save the license name once + fdl.append({'id': name}) + dc.append(name) fd['licenses'] = fdl cdx[purl] = fd else: @@ -137,8 +138,9 @@ def parse(self, data: json): fd[field] = d.get(field) licenses = d.get('licenses') fdl = [] - for lic in licenses: - fdl.append({'id': lic.get("name")}) + if licenses: + for lic in licenses: + fdl.append({'id': lic.get("name")}) fd['licenses'] = fdl cdx[purl] = fd # self.print_stderr(f'VD: {vdx}') diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 21fb0a6b..31171182 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -100,12 +100,13 @@ def parse(self, data: json): fd[field] = deps.get(field, '') licenses = deps.get('licenses') fdl = [] - dc = [] - for lic in licenses: - name = lic.get("name") - if name not in dc: # Only save the license name once - fdl.append({'id': name}) - dc.append(name) + if licenses: + dc = [] + for lic in licenses: + name = lic.get("name") + if name not in dc: # Only save the license name once + fdl.append({'id': name}) + dc.append(name) fd['licenses'] = fdl summary[purl] = fd else: # Normal file id type @@ -128,12 +129,13 @@ def parse(self, data: json): fd[field] = d.get(field) licenses = d.get('licenses') fdl = [] - dc = [] - for lic in licenses: - name = lic.get("name") - if name not in dc: # Only save the license name once - fdl.append({'id': name}) - dc.append(name) + if licenses: + dc = [] + for lic in licenses: + name = lic.get("name") + if name not in dc: # Only save the license name once + fdl.append({'id': name}) + dc.append(name) fd['licenses'] = fdl summary[purl] = fd return summary From d2610b69517399b4a05fd9229ec4e93e0ec5bde8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20P=C3=A9rez?= Date: Tue, 14 May 2024 13:01:27 -0300 Subject: [PATCH 136/489] SP-703 Enhanced token usage in Repo Version workflow (#40) --- .github/workflows/version-tag.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/version-tag.yml b/.github/workflows/version-tag.yml index c327eeda..895068a7 100644 --- a/.github/workflows/version-tag.yml +++ b/.github/workflows/version-tag.yml @@ -17,10 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - env: - GITHUB_TOKEN: ${{ secrets.SC_GH_TAG_TOKEN }} with: fetch-depth: '0' + token: ${{ secrets.SC_GH_TAG_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 with: From 4d276e9ddc5c13534287a8d22cc70db23172a260 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Wed, 5 Jun 2024 15:14:51 +0200 Subject: [PATCH 137/489] SP-802 Add file list option to scanning (#41) * added file list scanning option * updating action versions * removing error when no files are scanned --- .github/workflows/container-local-test.yml | 8 +++---- .github/workflows/container-publish-ghcr.yml | 14 ++++++------- .github/workflows/python-local-test.yml | 4 ++-- .github/workflows/python-publish-pypi.yml | 10 ++++----- .github/workflows/python-publish-testpypi.yml | 12 +++++------ CHANGELOG.md | 5 +++++ cert_download.sh | 4 ++-- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 8 +++++-- src/scanoss/scanner.py | 21 +++++++++++++++---- 10 files changed, 55 insertions(+), 33 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 1e20ae8f..dfaad526 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -19,11 +19,11 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Setup and build the python package - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' @@ -36,12 +36,12 @@ jobs: run: make dist - name: Setup Docker buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 # Build Docker image with Buildx - name: Build Docker Image id: build-and-push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: false diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 10a6bb12..42e0c9f9 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -22,11 +22,11 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Setup and build python package - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' @@ -40,16 +40,16 @@ jobs: # Add support for more platforms with QEMU - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 # Workaround: https://github.com/docker/build-push-action/issues/461 # uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf - name: Setup Docker buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 # Login against a Docker registry except on PR - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -60,12 +60,12 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" # Build and push Docker image with Buildx (don't push on PR) - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 7ccc05eb..55931935 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -17,10 +17,10 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 5ef8663a..a8d36608 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -11,10 +11,10 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' @@ -49,7 +49,7 @@ jobs: - name: Publish Package - ${{ github.ref_name }} uses: pypa/gh-action-pypi-publish@release/v1 with: -# skip_existing: true +# skip-existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} @@ -65,10 +65,10 @@ jobs: needs: [ deploy ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 9061225c..0c105c7f 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -10,10 +10,10 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' @@ -49,10 +49,10 @@ jobs: - name: Publish Test Package uses: pypa/gh-action-pypi-publish@release/v1 with: - skip_existing: true + skip-existing: true user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ test: if: success() @@ -60,10 +60,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10.x' diff --git a/CHANGELOG.md b/CHANGELOG.md index 224e69d3..c27b074d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.13.0] - 2024-06-05 +### Added +- Added `scan` command option to specify a list of files (`--files`) to analyse + ## [1.12.3] - 2024-05-13 ### Fixed - Fixed export issue when license details are missing (SPDX/CycloneDX) @@ -326,3 +330,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.12.1]: https://github.com/scanoss/scanoss.py/compare/v1.12.0...v1.12.1 [1.12.2]: https://github.com/scanoss/scanoss.py/compare/v1.12.1...v1.12.2 [1.12.3]: https://github.com/scanoss/scanoss.py/compare/v1.12.2...v1.12.3 +[1.13.0]: https://github.com/scanoss/scanoss.py/compare/v1.12.3...v1.13.0 diff --git a/cert_download.sh b/cert_download.sh index d9ef9f7d..82b3bc83 100755 --- a/cert_download.sh +++ b/cert_download.sh @@ -26,7 +26,7 @@ # Attempt to download an SSL certificate from the specified host and convert to a PEM file # -script_name=$(basename $0) +script_name=$(basename "$0") help() { @@ -47,7 +47,7 @@ VALID_ARGUMENTS=$# if [ "$VALID_ARGUMENTS" -eq 0 ]; then # No arguments supplied, print help help fi -set -- $OPTS +set -- "$OPTS" force=0 while :; do diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 499996e5..ab8834b9 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.12.3' +__version__ = '1.13.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 00797930..5e1783d3 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -72,6 +72,7 @@ def setup_args() -> None: help='Use a dependency file instead of a folder (optional)') p_scan.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, help='Scan the file contents supplied via STDIN (optional)') + p_scan.add_argument('--files', '-e', type=str, nargs="*", help='List of files to scan.') p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file') p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file') p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -445,8 +446,8 @@ def scan(parser, args): args: Namespace Parsed arguments """ - if not args.scan_dir and not args.wfp and not args.stdin and not args.dep: - print_stderr('Please specify a file/folder, fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)') + if not args.scan_dir and not args.wfp and not args.stdin and not args.dep and not args.files: + print_stderr('Please specify a file/folder, files (--files), fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)') parser.parse_args([args.subparser, '-h']) exit(1) if args.pac and args.proxy: @@ -556,6 +557,9 @@ def scan(parser, args): contents = sys.stdin.buffer.read() if not scanner.scan_contents(args.stdin, contents): exit(1) + elif args.files: + if not scanner.scan_files_with_options(args.files, args.dep, scanner.winnowing.file_map): + exit(1) elif args.scan_dir: if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 674fce9f..449ee3ec 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -522,8 +522,6 @@ def __finish_scan_threaded(self, file_map: dict = None) -> bool: else: raw_output += ",\n \"%s\":[%s]" % (file, json.dumps(dep_file, indent=2)) # End for loop - else: - success = False raw_output += "\n}" parsed_json = None try: @@ -625,7 +623,6 @@ def scan_files(self, files: []) -> bool: success = True if not files: raise Exception(f"ERROR: Please provide a non-empty list of filenames to scan") - self.print_msg(f'Scanning {len(files)} files...') spinner = None if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') @@ -637,7 +634,23 @@ def scan_files(self, files: []) -> bool: file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False + filtered_files = [] + # Filter the files to remove anything we shouldn't scan for file in files: + filename = os.path.basename(file) + filtered_filenames = self.__filter_files([filename]) + if not filtered_filenames or len(filtered_filenames) == 0: + self.print_debug(f'Skipping filtered file: {file}') + continue + paths = os.path.dirname(file).split(os.sep) + if len(self.__filter_dirs(paths)) == len(paths): # Nothing found to filter + filtered_files.append(file) + else: + self.print_debug(f'Skipping filtered (folder) file: {file}') + if len(filtered_files) > 0: + self.print_debug(f'Scanning {len(filtered_files)} files...') + # Process all the requested files + for file in filtered_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break @@ -697,7 +710,7 @@ def scan_files(self, files: []) -> bool: if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: - Scanner.print_stderr(f'Warning: No files found to scan from: {files}') + Scanner.print_stderr(f'Warning: No files found to scan from: {filtered_files}') return success def scan_files_with_options(self, files: [], deps_file: str = None, file_map: dict = None) -> bool: From dc96218ef5009676bcff4fc7a2ebb4144e8b058c Mon Sep 17 00:00:00 2001 From: Norcal Date: Wed, 17 Jul 2024 12:53:25 -0300 Subject: [PATCH 138/489] change pkg_resources to importlib_resources --- list.txt | 25 +++++++++++++++++++++++++ requirements-dev.txt | 1 + src/scanoss/scanner.py | 9 +++++---- src/scanoss/spdxlite.py | 9 +++++---- 4 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 list.txt diff --git a/list.txt b/list.txt new file mode 100644 index 00000000..2e4016c0 --- /dev/null +++ b/list.txt @@ -0,0 +1,25 @@ +CHANGELOG.md +CLIENT_HELP.md +CODE_OF_CONDUCT.md +CONTRIBUTING.md +Dockerfile +GHCR.md +GHCR_BUILD.md +LICENSE +Makefile +PACKAGE.md +README.md +SBOM.json +WINNOWING.md +cert_download.sh +date_time.py +list.txt +pyproject.toml +requirements-dev.txt +requirements-scancode.txt +requirements.txt +setup.cfg +src +tests +tools +version.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 0187cb26..87752de2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ wheel twine build grpcio-tools +importlib_resources \ No newline at end of file diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 449ee3ec..42574574 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -25,7 +25,7 @@ import os import sys import datetime -import pkg_resources +import importlib_resources from progress.bar import Bar from progress.spinner import Spinner @@ -270,9 +270,10 @@ def version_details() -> str: """ data = None try: - f_name = pkg_resources.resource_filename(__name__, 'data/build_date.txt') - with open(f_name, 'r') as f: - data = f.read().rstrip() + f_name = importlib_resources.files(__name__) / 'data/build_date.txt' + with importlib_resources.as_file(f_name) as f: + with open(f, 'r', encoding='utf-8') as file: + data = file.read().rstrip() except Exception as e: Scanner.print_stderr(f'Warning: Problem loading build time details: {e}') if not data or len(data) == 0: diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 31171182..94c9abc7 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -28,7 +28,7 @@ import datetime import getpass import re -import pkg_resources +import importlib_resources from . import __version__ @@ -300,9 +300,10 @@ def load_license_data_file(self, filename: str, lic_field: str = 'licenseId') -> :return: True if successful, False otherwise """ try: - f_name = pkg_resources.resource_filename(__name__, filename) - with open(f_name, 'r') as f: - data = json.loads(f.read()) + f_name = importlib_resources.files(__name__) / filename + with importlib_resources.as_file(f_name) as f: + with open(f, 'r', encoding='utf-8') as file: + data = json.load(file) except Exception as e: self.print_stderr(f'ERROR: Problem parsing SPDX license input JSON: {e}') return False From 2a57b0aa69349e2ab6ebc4ba94129c4247e024a2 Mon Sep 17 00:00:00 2001 From: Norcal Date: Tue, 30 Jul 2024 16:44:01 -0300 Subject: [PATCH 139/489] added support for python3.12 --- CHANGELOG.md | 5 +++++ requirements-dev.txt | 1 - requirements.txt | 3 ++- setup.cfg | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c27b074d..6e377bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.13.1] - PLACEHOLDER +### Added +- Added support for Python3.12 + - Module `pkg_resources` has been replaced with `importlib_resources` + ## [1.13.0] - 2024-06-05 ### Added - Added `scan` command option to specify a list of files (`--files`) to analyse diff --git a/requirements-dev.txt b/requirements-dev.txt index 87752de2..0187cb26 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,3 @@ wheel twine build grpcio-tools -importlib_resources \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8ef40e42..da90a28e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ protobuf>3.19.1 pypac urllib3 pyOpenSSL -google-api-core \ No newline at end of file +google-api-core +importlib_resources \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 261d39df..cdf17fcd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ install_requires = pypac pyOpenSSL google-api-core + importlib_resources [options.extras_require] fast_winnowing = From 4e8fd7390a1c14c3c04e650e5bf8c91b4bd3f863 Mon Sep 17 00:00:00 2001 From: Teddy Chung <112674394+ChinesewordTaiwan@users.noreply.github.com> Date: Wed, 7 Aug 2024 21:54:31 +0800 Subject: [PATCH 140/489] Use repr() to get an unambiguous string representation in filenames (#45) Co-authored-by: Teddy_Chung --- src/scanoss/winnowing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 985dd858..76704961 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -307,6 +307,10 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: return '' # Print file line content_length = len(contents) + # Use repr() to get an unambiguous string representation + file = repr(file) + # Remove the surrounding quotes that repr() adds + file = file[1:-1] wfp_filename = file if self.obfuscate: # hide the real size of the file and its name, but keep the suffix wfp_filename = f'{self.ob_count}{pathlib.Path(file).suffix}' From c1662a5f9eef4706c026c586895ffebc281271f3 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Wed, 7 Aug 2024 13:14:24 -0300 Subject: [PATCH 141/489] deleted list.txt file --- list.txt | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 list.txt diff --git a/list.txt b/list.txt deleted file mode 100644 index 2e4016c0..00000000 --- a/list.txt +++ /dev/null @@ -1,25 +0,0 @@ -CHANGELOG.md -CLIENT_HELP.md -CODE_OF_CONDUCT.md -CONTRIBUTING.md -Dockerfile -GHCR.md -GHCR_BUILD.md -LICENSE -Makefile -PACKAGE.md -README.md -SBOM.json -WINNOWING.md -cert_download.sh -date_time.py -list.txt -pyproject.toml -requirements-dev.txt -requirements-scancode.txt -requirements.txt -setup.cfg -src -tests -tools -version.py From 8c0bbf6582544465c1f77973a9724c41cd191624 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz <166400360+scanossjeronimo@users.noreply.github.com> Date: Sat, 10 Aug 2024 05:58:22 -0300 Subject: [PATCH 142/489] SP-1260 Fix UTF-16 Filename Encoding (#46) * changed line 310 to return a utf-8 compatible version of the filename --- CHANGELOG.md | 6 ++++-- src/scanoss/__init__.py | 2 +- src/scanoss/winnowing.py | 6 +----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e377bfd..a1ce2c91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.13.1] - PLACEHOLDER +## [1.14.0] - 2024-08-09 ### Added - Added support for Python3.12 - - Module `pkg_resources` has been replaced with `importlib_resources` + - Module `pkg_resources` has been replaced with `importlib_resources` +- Added support for UTF-16 filenames ## [1.13.0] - 2024-06-05 ### Added @@ -336,3 +337,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.12.2]: https://github.com/scanoss/scanoss.py/compare/v1.12.1...v1.12.2 [1.12.3]: https://github.com/scanoss/scanoss.py/compare/v1.12.2...v1.12.3 [1.13.0]: https://github.com/scanoss/scanoss.py/compare/v1.12.3...v1.13.0 +[1.14.0]: https://github.com/scanoss/scanoss.py/compare/v1.13.0...v1.14.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index ab8834b9..217cabc6 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.13.0' +__version__ = '1.14.0' diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 76704961..9c21930d 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -307,11 +307,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: return '' # Print file line content_length = len(contents) - # Use repr() to get an unambiguous string representation - file = repr(file) - # Remove the surrounding quotes that repr() adds - file = file[1:-1] - wfp_filename = file + wfp_filename = repr(file).strip("'") # return a utf-8 compatible version of the filename if self.obfuscate: # hide the real size of the file and its name, but keep the suffix wfp_filename = f'{self.ob_count}{pathlib.Path(file).suffix}' self.ob_count = self.ob_count + 1 From 2e575a2bbf9f9bfe3626dc363bfce7cc5a3c324d Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Wed, 14 Aug 2024 11:40:21 -0300 Subject: [PATCH 143/489] SP-900 User-Guide-for-scanoss-py --- .gitignore | 1 + .readthedocs.yaml | 32 +++ docs/Makefile | 20 ++ docs/make.bat | 35 +++ docs/requirements-docs.txt | 1 + docs/source/_static/scanosslogo.png | Bin 0 -> 31268 bytes docs/source/conf.py | 28 +++ docs/source/index.rst | 362 ++++++++++++++++++++++++++++ 8 files changed, 479 insertions(+) create mode 100644 .readthedocs.yaml create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/requirements-docs.txt create mode 100644 docs/source/_static/scanosslogo.png create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst diff --git a/.gitignore b/.gitignore index 8968fc1c..f6ba5eef 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ bad*.txt *.gz *.zip local-*.txt +docs/build \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..a6d1391b --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: '3.12' + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/source/requirements-docs.txt diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d0c3cbf1 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..747ffb7b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 00000000..a95ae18b --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1 @@ +furo diff --git a/docs/source/_static/scanosslogo.png b/docs/source/_static/scanosslogo.png new file mode 100644 index 0000000000000000000000000000000000000000..5cfb6e5be85446111cc4352c4bdc5266ca74a275 GIT binary patch literal 31268 zcmZ6z2{@GP`vy$JWSz+_+t6Z56p@6cK?xyAQdwGT*+P;m*#@CPB*Ms&rAQ(YSyBnf zR!NGAC?zREzVms1|L^#Y<9m;{cVXt4=eh6ezOM5+&-0oXb2B4eE^#gv78c%JJL#4z zEX(kp%UDPp_}8z1p1&+CXC`*hw_63Uetonew0HX5?9`t>f3}W3f7tVNG~=qm-gc92 zMaA`+DR-j>;;YvjxTCcDR`k4)NaER;A2PddaYYL5Wy*H)=B&(9jyHDgFloPgJ?Z%8 z=NZ$BLvfrg*IfIi=N}yAqSC|rN)BwwEhatgeAkVB_zkYq6dOJ@;X{GjWv+WAqHL{Y$IBaEJkAL#%jj;vh3crhU4}XL*Iu)g@vuNE8hw7aA*pOh@{Fy6l&m6Bz9f4zS+fpyjxFC^59c8N0NsN-+ELD zXw@;9%xg^_w1mc%HOq)!_OY*Y5G_9c=>57VhmtqOg=8<~PM8^*x5&3F60v4u5U&3F0D8px_fQc~p9wKA?- z#ltb=rzMf)`TEWp23a_VEZe*#v*Y99{2%@cF)?DxezB&@Hs|=WexD0^*kYQler96+ zd;JDO200(ERCnupj)jmk?Us<>;oxLB#KJ}`+Qa5{x6e}{Pk3(eU*hiFyV*z-njkGA zw5P|a)+N0zlsnWxemUd%Ju6~0=SSuUX*KTG>bRM zVxEtMJFFkq6A==cBQq(Mq%Ovdyu8S3`LCsCEBXRS+ zNZggWRBhP)(9vyDor)38q4>QF5nGS!e{|YT#D)DPbnX= zHMK4tetv0oyPK6#ENLW4U{R=n@y#z$4j(-SYbv>@*Z0`&nF`SPPNnniC0XpSz3c3doSe*q zFuyM}mOy6am;9Uy;-?!K)gQL964YmiXOo9l-yfV*Eu9g4Zz#h_rJsC#RP=Fp!>^MA z_x8y&?=mNOpSQHMY&7kDV5Kl4BEv?dA5y?`WLE}1{%>;xa0RWrf&wF#(%Ua=_poP> zsN=g={J7W`=xSkciJzJfPGh+GXlGtBzUdZP&k{qTipN^!^?bBfoei+C=wgj1VBR z!4rL@`$*oRTvWciw;q4+6yf^2Bgpal!d%i%yARg*sbo`zH3ex4QQcp@GLr*ISOZzB zpk;4|rvk3XHFx4S2W@R_uQ-T?a7M8qg*@r99UXALcd*K1&^EM8C$dbvYC!VH;lshF zZERj?gi2EBg~cU>g#kLUvcsOlZpW1=N0zNF=yp-ed(5)LdqXCoEd3DT>-N5V`#P^& zxxzudl1sia;&VTcWp8LVnPxL+o7H(bvwvcd?4ZEqAXZ*pep5|Njh|Iy85cETHpbex z{dZuUK;28K(y_3$71xhYk0iNxRMpn? z1&Vu#(?}NEvakt?ii)<8qyt=(%P+gTou0L|eMdUzU$z!+4?4g3+Bb#zp^$^#-sSuh zA6&oM?$*DCP5K=8tq)iDqxbuehq)>FY9N!$!%?7^XDAav_S1LU##x>`Ezd5s^O+`zO{LsLQWqZui#C6mc#UVYy18(nnI4 z*-7BHb^Q~EqMcPu=;6d6`1j4?TKjJnauk^s!N;kpgVp?a;&%_YC)f`;VA<6{an zJ*wsvOJ~~jC1@lENeknf7Fdb1=X8gR>2>A|?bP&iagU1E*I(EkiA9(W_?FJ}Vy{0j zw2#S-CwG4_*~~L?(q5ECqH2pqb!}L?_OoeLXO*V`+p# zh5g;k{Nh%!AB{R5G4M?)zcP3Q!;q0AC?>Yc(NPxZX!YvVnJN#LVza;Oj~^5ftkickd?OxwA2%kX1LY#{etjDeH>G(@t&Epv-Qy zmuXguh>lLWe%-9%c)QnqJ5-<#zS_HX@0MG?KK{my)v|JOE<5=owDetE6mTGbG4@(n z(Y?LZaHXK7#Xm?bDY3C!5rt9Nmg9Ymjg8UdN@J?4sioy*F-ghA>Y?xTT-4wl3I;wt z8`s<3mD#%W+Sjk{ckbMI^8EQ~4Gp0@!5q3<`|oc9bN}YQzrO1ta81+m_{igUK|w)t z8yiNSI5oKKgO^SvP|E~r$f3wWAubCg%e{MZ-8T=k*-lVNySe%K*Wt1c9_al1`SVFz zo1CU*YGI)~``4lRtlRbf(r-Wf8@r)?;WF`uchzQ;qlB`}c|8mIGF}ee-czjf_`MAA zQJ?#l!-BGv8ZCL6vX8Z0@f~{+w)y0n7v)YyxGKsmW1O3PSYm7y6Swsv1pcEH<%esvkKFJq;=6Y3$~gV*43D&K2Eq+M#j}3#%9c}~RZmaa`G0*C z=4|zEd@5jahlEI^mWoR1gvxsldiaU-wdel+a`-jY^=rJByIY+yY=iw-`rBV0@F``# z&7GqwP$3zHA3UWJM^)&R3iQhRcDK%kZ6))jw|#Q__~g=1<-|+s&+1%XqDds-@3v1htE#!X-qQ^YHwMjmJ-#5e!_172)^o4@ zn&$M^iq4IAJmi2-=ZllS19XzeMtNkTPgoPSNh5W6MC7rrugdLwo~}`MdC5*4PwyOc z`uzN=&Y!oZfVgDDGnPiNCI%9hXcCtOzCAqPwoR_Q~fEzfP1h*|y4~y|Vvs+l!nXYCLRIaq&_O&$=1m zM4g|cBO!|#Ob2u0L1c|tJ;o0CW~!Y^zq~z%z0CMNY+h6Ac~q7)Qf6e8l;Tzhto7>HGHQ>NSNFGd z6IQW%(!{T4vVA)%^Ih_)C^fJD4%EUg{N-by!qCrZT$sNwPlDA z*&akbRFJS2HNM_gvx(W^`|VB2*~$CS zZ{ONQl+}1XoQtZRtcgSVksI&fD)q}kC&y-!+{jGde9|@U!i9{2N~zdF0gtL{TPLf- zb9O|_Mrggd=K1g`vczr&2P(7U!2t^k?@0tLo6W@-gS`BFT`>~73W`+iV24J(T~p7^ zy-(ZPc96UatxIjyTzlm2_7T~aJ$yU8)J;fA>iQj&2hDPs$mKXW#QCng<<@`e?b+{j zA3uIXvwA&#m8MUtR&I+D+2qRnT}f*f7Qcs#-m0CH zRZv(#rO#-be3PuV_+`)0>0VT$kW~Z>!@zlZ5D2|@{2ca+_)Fzpv>M# zsXUGprn9s#?K}BaEKiWe7%50*qBqG1`8OvhBvkIu#6d4r3(fJbT`WcIleCFwl(kdY zuUNM1iGEKk)_=10yDkT>IIz<3^@ks%4S3dPN^bR+ac%!&d{YIbc54}n0QXK)Q(jGA z4;g+7rH-xNb&hvr>_R9}Xgv>v*W0T4eB2ou7dQ9$nkTtZR%%R2Kw$cE=V-BP^Hm%D zprD2B26KA6+shj(5_SK|kF+MP)6tQj(!Z_X=Rffxd;7=FpYLFGh^N$0Ii&FE_gv6i zgjNz6b@3M4q<&9z@Zw*!Q=d(={l7BE%*ng%uP;w`>Sf{Zm!FwFPLH1;i-_zWp9$Pg zX*8jwp~twHoW4AbAub>+{CMc!Z_UY?2^D5;F7CDN$&;NO+y2-A=Nxz#w1voh;(q=n zCeJr!b^h5#eYTp8P4vy{St-M3;pF4vlcwP&d~W`?0m)+NO_gV7Lm1nEN>?FdLWa1L zVjc%T3DGC$(k|{Ip}BG}zG{r$uBpH#aY>U|%*W<4X7SRp-}RQ0+f_cKzmSyi|IVQ^o7C?gIYx z2k(7c{4>=0{g0e$1R#m}VD0Je2O~t2$Robmo%|f(OZq(woQUI{*G{w_0bcdF0(2G-_XyR41#z>myDuq2Uz$_7{nUNd#@pe&$ z!7Scy&9;71Ki-~h05C@*O@&}Jx1i<{r$Z`{6Tgm6Ogu7z%>ROsrY>t%u!&;BPM1sA z{rz3tRo!#i*d!9Q`8$$}n)@r&zS~DS5iI0ZBe!$Maf2qe`_G0NHr}Ab#K&jy`5H>3 z5;*JT-h!2YoMf6r%%I1Ku@^Z(e+Eyxzba$(wqsWIwWEcS5MPmG*odx?WfJeXBxke{ z>+imK6}oEWe3y-0@AdC_4t4yUnV|F*B)du?M^$!?&R;gv?;=$XPO_KI?APDOd}A0d z-h#uHXvsyf{XIF@hWy~oONQYgQnXxOAb^-%}n%9 zBg*bz8N}t9RG&O~BC~E?bVh~*5~g$iElF^DjK^OPcAst*T&G!OS-RRgIDC*5S2bY! z>|g6t**6j9ORu~jx%CPPIcJyGw}Gm}J$j2ot|~*ZLp1C;b7o79%9brx6C}63yt#Ls zw%<5Cnop9%&AmgN@AS|)&F*~D9aaN>q8#oH-aC0Mk;AwvRKxS#LdZh!k{`qMJ|OD- z?+@qAjj7@ZNZ(VBIHk*QT2a7T_8&QCpeoutF2To%&n*DOM6@(!nc5CjfUik?>Gl6+ zZa+^`W3$*MBqWsh?d|I*ex}x|BJ0vC*+V;krZ$Po z7_foJFl|!ReQA5zP@)^q^2~6|I#nJv3%#PEqS@Jz#HXE|qkt;8H*T=%`%Lr*6FI7SbT;MT5M!Ox3`-%)Y-Nee`yN~L8#l!@9NG{Dx1FfZ^Ge-;t``2m}Qb@itpUHj7};8iCDJilWk(_ z^lIJ<%rNi;yW-;G7r7Wl61_w_iC|zb82UY#z@_I?4+q9ny$g)ncJ{Z-xHTe@$Om)X zS^wzAkFRA0C#R-PkLoR5#U{%nxMX#XwxMutb=DW;z0me4a78uA+X~_P839^qN$)MlNiset({u$i7JGjG9j~fkX-?;P>!YvgHP0Vc~>)Rtt3* zmDFO+AG+COqmMrKpLBMvL$7af_HLi@!Fv~8SDdIg`gEnDk`je?Z%~j(*5kr_n||!X_vvD$*oUtc_S4m-czeZRg9h%~8^D{p6KhXTDPA`GbsFIS5i>;5VP+3`q3wYflM1TmeF8_rD5NZe0AdYO* zyGBLDBOUBn!S$=^ELi!Fs@Lx4$VTc8jqED5UpLa*3GM{E!tlKT52wYyGXypsQ`oTK z#HVK|4}LUq2h<$~kHNTbf!^}~AAq$#aNlRmFoU@e9P%oRjZ=7Ur^)yMLqkKQCe`bl zCe04K;_8g0f8Uo36(Y6H3Ntq4OB`!|H zVaeoR?ew>*Axm@fP%WKq^*uxbu!tt+YrfRgo{F)tv4XJ8j3(EihugXf_OyCb$(eN! z?9arJrDu3sNB@zK_&#xmZ7C@!1gCLz(+Nupi|qg$-dYljbnGpVJ`qC%I_7Ir=V|&h zJ$E<{mzJv<801a6pj84iefZqWzTuG(%DRrxX*JXzfO_qRv%}nX`$!fC4uEV4YW(`T z0$g1_DuHNJnAREp+DzSZe=lZbbprf_YL&dLuC88#8htU)(K5se5T;FEdzifWT0Ej8 zDJdyjOAWXcDJtB19fDTZfA_lptx>2``aBiJl1nl0Mw%Qu(t1518Dr&@#5R$=w7OV z!k3%KBkVVw*3QUiYX>T;3VRdy_WHTs2LQsDe+dp_VN_2HB?`@!0V4@rqYYwedbG0u z_pk2xPQrV*LucCL-I-BI|H-#8T69Ml@~d&2kb;D(J>{Tip;EZ%OAO0&b#*=X`Ali` znl;oSPXw=B&{WIwtD89MhZ4ye$B@4tpnE5{HnfNWLPC*R!Pp5R(B8P+{_tm>674Jv z^w$gYSGZ%XpR%_0UsurKj2${ZpU9zE6|x}0iD>st(p1BU$bq&~=D>FQ!A~x;f+SSU z<|(%~cNMN)y{c?iP4;F4*w=QB9yE~Lcyk9}lcmCpI5}SHNt&8dHF|Aw?_P8BhBc{n zjjKE#YzMezysxKZeV}nAV_3^=EPG!#ja7J$3z7LHua89@3S;A88=5b znKs;KgrhnA@qAOKeZ_0CkqC)BU}38DKKdkiq&`NCH9n;H_Cw>gE9O5RIuEig)y zQa*4eB6I6$S-zD{qK^;X?w_alHQz`8Z$@`1J+fSuQV*yznwNT5cFmgQ-o9v!2v+97 z-|=FQ2@!=GfRY|eJzfP?=0&a%KTsJn5dQ$M{m{GrNM2D9m-oelsi~s89SvV!@q}x?T|A4viOYYMTK7 zH#3k3Nl1LJ9-P@UEb|RXF>4kCWl`a=wSF?7-Cgb3hedFLKM_y7apOww06>Xj;MpE5 z{vIH8UEF7F{qy&L=Uy9|owpCymA!UnI8D5(+j_h1w>UnJaim!^s=R4*y7b1Ct37v~ zKQ}RgzAVHrSa5untN{L()jOaqbpuhE9I*+YfP&ETC?Bdx)?4_%Pf(@_DvFA+c%QJc zO{mt+f!W#Fbx^B9ZI4>Ep}?r@H$D_G+akR*YxFo#I#KHZPIBcuxL)n`XAwMDrIUo%%!six) z2{iKq00p`-g;|c2J~GpPWSxRSjQ_Vc)P!3gcAsZ%O_dIwWu1!qiiC!{yPba}t%q4w zR;C97Loua3;NvAD=k9#oVS6Nr3*s8FK%1OY8p8VH;OU`b{kIMgoF$5Aw1|`xArYkM zh3F28tO%|TX-Cy~m_uA{t!es#fR|jby6>3zT9dR;tyiDhuZgKSvJ+ z9pJl(B;MrK#auoVJimVV^5u-IwEU}IkPvfmot3F=Go0_2sKGrqb_pDRp0@qy(bdEI zi}&9?4h|FyBO8_0ix$P^p1T+u2an8u8%{K3=$fE`P=k|+RCud%bii=E>diHoOgaU%lf^%~Ot3w&O44z`SFyQYR)7)R7+ z=7MgUbDOw^4Seh9v(@8}XFrkQ&2f6rPsCWEc~AbHtPWGnK{^7MX48U}JOHr#3qngl zL4mQO(qk~4fOn0JwB|%xs5I^G>O+=1T+?=~XT-(T9&{UA9)|7-6a}Oonc$zz0(*W` z)i3gvYS3(mMFpNH0DE_%pg@iOZ?Hm#xE0*g5Y=;G;K;?c0)ssVfz&;R$z(BU3(ST*~1?Z#f^y6 z1v z#HRyFp1>8`*;kTvX5WY3^%xW{?zTl+^hXs0ZUng=1<~eq7wshDwe39ma&J&{GWIWT?9xHkO``G%Atz3hRyhyc zTMr`IXrX9dv2_rVq@b{{>`t`!HeY4}>+_;SdrFEaY57l)q{u(OV5*&bqYlrBwS~o_ zYVQ#O!BaS|sk^Mqcahb1J10au^o{)dd^R=;5%4H!C6iZATz9~shjXA5aI)N~^$udT z`8L5fM~WO~%ywlXXZpG_w)3i)PWv=852M$<|0VxgSq-o(@zfw0A3~Vi6!^=Ti;Ig~ zU-Rk8K}JG?q5Z#sydHA6_{Fp|=Vhs~)xY1k?b^As;o0ZTtj?2FgHh18nx|gZoz)aM z7yeRSZ79GaF)<$cuyE1)Jy$Bly4ct2@{NT@f+M@u2CY`t!*T# z>*H78G}(JlnNVYnSbWMJR7vEr>`pQvs!-!ov4=+Ehm1 ztuZ25RDABpY;#K77RA!7ejl%(yb86*SRJ0E$r0Dq*5(Ax!=@J}s~KAjT{{6iUQ$+8 zVkqMK(zR=G?;Xu9KiUPAvVirB&0ApT^Ojsnf#Wv|aAa_Cv=?vDAeS7emnz9fB|ijC1go+rrAxs<3U_HV#onqpP=`S0^l?jfuX7XczJhOHB}{5DShOqR|dQkGBV5JVlP}cifV526c`}j zeHbS=RUND%9zG0`bBQbPH`1G<1I73ztHiqSk<%)|o1rHG8<)p|1n{W&vXu*4 z($ahE>?9!SLtuBR8bIw^h#vZmR?!^LzH_u`V&=~2?}v?zjpecU=x{(E31xLgJI?}H z<0yVaJM{Q?g@Z<^Z6zVn1_ArnSY`CpXqGWt?|4hx?HhG;AnsZb8%-SW;Ckt{S0m482mucW)q->zXyEgZ9~`9p0r2z zDX-?+P$6Qs{`;_3(qRTxN%5|(eI-}XH*MjG0|S%c*LT3C1&cEa#+owIscs6CrVpU- z1&st4cWpT7a&E0&w+^`41V5hB1NH`<7YRbrBCL|&uZ&n+4cb^DR*B{M`SB^^uK;Wl^}&l{uPfF; zyIxDBhD9}dRCWK4tZD|0u^SEvrTw>#f*ewYs)7}vj$;c(q3Ru*e0yI$uWV(LYBer< zRp-n`;S@`2Yk%>w%dTY>rnVE?ZIWHmp}ejvt_F>z<@KJ0>493V=KMm2gJ0^o^uHgR z;IHBhY-~i)CfzvEdHpS9x=O%i&}}a7hsVY#t%05{aybH%|~C|9CTlLdXZOGSId|!6axsqW>n&59rV1;G#i! z(g%s#8yl+EyL9Q2%`TXskmHU$y}ZWelBFxzOS9MBk{77tCMxN z`Tftf4McLVs0Faz0`2Myju;`6f*IunA7Y-HEW2q_(xY%rqN;&8eFdgdZ{Y_U)BoF0 zy>zL@w!mKvu0TxzNCahtcTtcF{(K)=Vr#X}6~_tI*LT>1ZS)T|}NKTGT>C0fPCK6Bbl`D?h z4fpNqWwGaI#d9Ij?z1_^FOQenmcFS|C&|3Zb4Zt9L%;)5G&bn!UPp&^3Elo>P;3Hl z((JpMzY9J&v2B6sI8=^cFbQ_uVj2fdDlr;G#$ipAgE>5Y$?BeMzh3Mh8g@pykeJw0bWrWkDZd~X`J&mT1o8wS{}2EJ z4FYOFbm!=fo)YMgFYC^pe(RxWeK{lJIjVLj&jn7Y0G3h>>y2n6c$ykw&p)z1lJMxGe1_SHYRmBz4GArT^t^jRU9$OUNKQYo z=0GuExSxRr6@|JV2{eUXUPilfrA-eu|Im$=;6ew*~;! z-A5__D6%-q!6Vx6GH>^nzS330&B)dSZOg4B239rS#StD`Jug8HqkaMI@t1kr=y2C0 z3yFwq@_eV$9J6Km`KAMRohir=hCPHs3y`cGB6+wam$xHvd`ea$5rUYLL?Zq3UY;kR z;hXRf`G!M;09-vhdydH02yL6E zkXy?OJ=)VXD!kb_jIt|rg&2*F$(B5#s-fohP`;o)MQZ6HT|`R-GNA9Xp;w$a@xCF9 zplkrMWfO)3_UMGAK#!Ep6BHIagyS0~r>eEA)NhY+A&3L5Of%YRMD%~eTj4{3+70}& zMy2}wGJ4ZR_@){oxe4uGA`8UaMF@2ELNF4*A!2W_ZOgK7IdkKghVzh;Q|jqw*iomdH2GShmMzE92w&WvE-oj8x#ZoOvVejl-ByNb40`Vaec)N zdo`pv6vQa=-<=`<1T3~ed?DDYs)2atSB*_ggt~*?z{qM3jwUYHs0bZAvJ#s)()?s4~Va~#sQe=Jma62K8wO+Y;_348%)4;nhUwTv#))@Rey6`%|^8?_N z=qt3KbBQtV`>e{t*Yv^5_wjkGzz+Gd*gm{HJ^dMI&RwRa2I?xi8?OKiTd@!RJ8gn%kvpX&K^TJW zy7qE)RT~%?O*e9HxOnwyj8+X{TRYY!T#}6hMD&r|<49IVe{SWdhj-u#8{I(-g+?a1 z9V_a66l8kf7*+f9#NErB^jQrL4OuPV7;Zf8$YAHGG;U*hrz5 zm3tfsIj0B;03rM*xTB9HYBA%*<&=~bK%N+vbcP&Y5#9?*tYoJ`@AV5Q^0T!Vh=1d^*I^vxC->1XqxgB1P@^o-B zdVG*q*T62f?W<-I#-en#l*(k z81%nly~B@H#y2|fbF`Ny-d1~)+z6ltz_@Mrhwz5O7lbl` z|9%07-islc%?nYFJTW9_T4(P8}_APdkYH-lj}NSNDqCRP|rY%Mr-wK z{r+Hws_5#Tmey7RsP@X|*-o$GsQ=)lW$UCSsqHIcRR=1Mv15^-uQ3Baa=Y`|ogf@u zxo>WM1PW~q;aGN_Pg2`n!*;?hxaqz9eH#Po zd}htCd7Q|G$6SPsW%;L{t2j8@S{w-gD-So^Crg%-ImpYC8R5gaV+IdlnkLK8;uGf%yCZ8vJ4 za1x|Ofw`-XI+um%^9d(qX2!Q4mlK4uglDQ4VZ7EXgF&v>2>4--h9^qv6{mENeqqIH zcffcj4da{c!N-5V=`Ab_JE@U80Ksr>qAqP=oM=LIb)~4m-xIb@>MKOf_Wu0TR(>9^ zz-jw|h3D^2KtwIDS37g&Oav3ounWjVa>19uCBR)m8TKMfHSZsVQ|iHwCz3eABWnJ? zck)HcamOM3$_bu>7n5-$URsX`=m@RLF#_ux)7K8KLt}CbRW_mqe#jC-g_M>~p5TGgruyax?)6uB3K!R{>GbA%wX3@OzVinA&Y?Ep^f>xbvI3SY)vIN_KFZT{Ep zQB~kpT7DT~EIAh12CyOQ3JV01B(|0X76W+qaJLb6(DD&Vw!PYyhW#8%`IY`oYS=zH zi5gst4uHsRQ%SwW`vt}^ZxXir{+wob)(t1M0|ySkYJtI_YB1&$y#uxW>vL3;l(@N# zM~8U+nJRpG?6`5$CJK!)v=up$z;z8cA1nNDzHQ?B-n7@hp53J_4CjRzyb834nxdDM z3uxUo$Y;MCp^nY0T=>B0tID8A#*0bDuh$SVYP+I!r=q;pE?dxA7+ol^;#JVL*89!P z_0bW5O^PUjpA_v2kU+8Og{4lZ3>NyK(Zr>YVBIV+4-P>K%8Sk2CfzggyEiFy-chd5 zP}=&uq_X?Yy~X}NwH~L3_PJFwJ$rd)qef{TQ^#2OHm&_&GaJ7s2U$Ih^|*c7+`{0S zBc;dIa%oSWsh8s4mbt@S!ZH20I2Vi%M&0T#8)gs0k9P05x$xIbF;yk+c)!Ycqw~v% zSHSzXj_3_X^E~Q%utiZE23^8cR5U%m2>v%5P6Wq&HMhF&9NSG)YuZb#=EsTYITs0b zms)beamw)-*z3+#X@QcW;eh4FyS+nZsgwMA^xAn3N0o~nNCzpEh#&xxoQt0&97l}W&3B@iqD9o%lmA3ZTmDT9dA~Thb^MwbkLA^? zyN(@S``bt==joAuwA=xY9dWUZ<4{simle>O84{6ga|mybb$Ld;hov4=Z(l2a1F9H_ z(h;;u(=uQ*)9Tzb@4E1iAziNT_oLZ>=qq9bym=&JdkNtctYU5ryb(@*XpZ&Oo+1^# zIGOLVmX(9c1?)L3Law4FeK=bB^ZT|9P8{JG?i#hHlS0}xv~JbJCTqTHxRP;U-yyF@ z%jpINXkp9`Kuxhuy?pt+qdSUmyl=KxEZV8<46nTv=eE|u#>ckdAw!JENAdn>=ejB< zBgOi^T*Y8th_1Hq7OYvIB~mJ#yO#Ajr}tU)tFISp0l^_d*Sh)lubY3?X{m?2t4KnX zEiGRs`V+sycv__LaMsaEgooobL91QCC%c+-yBA@!jlBibfE>NrE93|3Sx&yy=7-G+ z)!10=LjG-916oiByb2pD(6`M23=9-?sjuI~xzZf%77t-pqVVaUua;0-hNm~bw=~tN za0YlcADaf&l=)u@Qr^7z3XDF2Nrh}N@?eD5;!TR1Ha)GM)PDZPw(-%wsgW(S&y;Lp z@?A1ueJNDY(~~YnHleM-@{R}B>ro>Ab1~C0L%tvW8O_`DYmT(ez^T>pzORDyOQ1+A zI3mkxAbDi3%<4=n9QgK|T@W6+2y)KzlAF|!S+8ZB{3|`4>$8k9C_TbL4npY3HWz_A zY#hIQ`Lg*gUteD)-)iQ?hzRxw(y@8z5Q*>Oy%EK48fkV_LkatQ$9q=vB4IoDfwM1c zR&S?EdS#QNo5ILBMcP}hu_djgOzMbH$W+~X@wulnZ*(u^M!OT$d{DW8a%5HThUm5X8EeXB|l0gb= zIf!fP50ItiHHVtH%=8w0txlu8`|rhR&XSDnRLCkMO04nqK(1niOjXwIQMtF7fC+_@ zpa5H+0rOdyT;OZYzKA`DAZhd-tg^g+bcOZEX>IS21+@<<&?S2@$W;Q3R=kBip|8$` zjBm&=-@Dm~!+Ib7@xPax!3`=aDsr_nj}b44jioUzuH>=&{qC&IJ}}^x6F!uVbWP8M zaUCgsYhg*rct@sRLjsL3n93&4Zg$VUS4J-0$i}J&Q;gS@v^2(ekgl|~Fr;HgcWwLZ ziwp)Sg0x0x_Z54;l-*az9V=I^e4);pG8d#Wl3PC~Zm&3j zHaIO!)avu}T-nA${T3H;*f_Yl{3NNg7|$k-l8&rYBP>nAZ3&oa4wTy4Jw*mND8bg#PxS+;bRL;`u$3j$)xPGP(F6^ zov=7EE>cL@7xY8DKL44wz)jg4zz@|}B}v&Ap(&AbXmJdEA={&?KRo~bCwC(Zq$$gn zu?h;j2xH-bC6+`{{MAgia!_}rYfT5}6cWp$1jm#7+H5vfLCoQVlXDu;92b>oc$!rg zm6Y7LcTYG4M47{}-ripMd~41H`R0ikS$bhv-i>Snqule1?mg@5Qc_cMYhVN}c|JOw zlV&UVgGok?Ayaqs$G8qj6?f<6#yAhAd+md7c6@kvxNIW^n;ai#n@7^%`lCf$9-HoH z8fVt{u{Fuq2?+}`Yi4Hy{&y`!;#zn$IcN2~J<=sA_dJg@UIn?}2UbRDIC-wRz=dfs1NY+OBMsLIp$8a%oMMv~P% z-ij)nsvVSnT+HSni17kg`?YR86qi$2(bna+(z7mw*qGNZK-s<`<92e?-TQ z6MA{FmS0Yd()L3A^3re3#z6=_7KZ?utWW=Vzujsj4@aAC?cm`k8Zb0PR`3fS=hKl^ zC!rWQqm{ik1wMEHxfb5~@8X#I`tRbL&n$N0ziAOgH|8hhicXzq%T%`p+hhE;=xjq#d}%b^2MaulL7iLlm{)!+sN_x^C&aJP(UFEKeGaXman7aRnb_+qv^-j z36_HjjUXvSZGVPwj)4OHTV1&;By^tKyz%;*`nhq1^o36wvR!OUfomuFZyV`@Z7i!n zf0&cz$X7lkw~h;-ze(Lx-E;@TYq^(-gh& zH#VKz?Q4PLN!AnJ>aR?w z55wd`MtgSZb^->DB}+!~(@_9UX=m>G|J2>-h%XTh&4|&IWaXGqi5?=0x4O zj{=)L-(4UZ4NuR_!Ur2OdjLe3%iC%->ngtqYb&Sp<0`mC>3o;Z%rqEb9Sz_*C;F9O;4 z@b9=%#H=T31u?eAXqtf-Qj97M$zHNIE&o0qZ7%V_g9nQZtb(|!SNBe?vi5ztCRK({ z6C8=Ft1EeU=?v6k)FDO|imErJo}B!!e7Q9>;w@iK-?z(Ki0=4sFDDIBp1lD zRROMM`td4HndW*tM*{LVm9HNTWZ5Z>dA$-z`ui&#o_T6-Ci8*~lR#q0b3b_8kzqXwDZ*d;8HRR}n z7+BbJqJv(j01Lawks!*H8}vyzOkou(DJfx^R|IXxz4#^Osp@ul+|wOQ%9hBhsa+z- z%;0aZa~=hAN~%PNd%+V)V3OHN_-L5t3A#9aZPCrr(!u8;U2_|t)vBOoJbm`8v51%n zX}%4Mm=A1`BTzxd13q6pLF~YAzq;uNEDJk`nq#;<66@ZA$^U)&(I@2Q zO9)ArIElfO#&lq!({H^w|07!C7z}$1qZQgC)}Rks35bc^4?FT=u98>Yxg}P}ruSIt zAw!t1ho8XCQ2p)Rfc%CHMmBLS>BJpCHd;P4K$zw+Q=(`CmqhHFh7Ds2G*i=d|Lm15 zpkE}45&y)*E@S8(va4aDR6Oh>Hm&HjFomEmF$hs6UeUV5ChWW-4#Vl6+}iaRe-dYT zxI^vWJ@=fF5}smlIL3#^9?pGUMOfIdMY0QZFz*v%t&ChWV@!-(7QP25g#()s`Wq}%SX=ZiVs^!!pzMT-kA=|uc1O<;ihKRK)Y@|xY%SYAS7 z%&^5>5CbZhrbUD>Jq!~hc0rLTvZ?l>=DRRu_R#6ghQZ+-p7^e#;P(dw?>cwMmP``P z(w;-e%c)0#w}5wJ?9B!DPuNn3X>vwQ9O!Sz{|0Ngg_ZR!hQ7Y5tGzOmeag4Z0B;sHB@q$!us39scId{Kdg0T3&u z>?IXns-IkLl!Hr|n?qeky=Z#odx#h)rAcJz%QQe-Ya(p_5Y#EW#rN)A!LIp0I{$CZ zju_c{3;$c;N4!rC?OWmPEnJzos;1Mc8sg3qrK`jP<5vF(YDcj5|0nV@qRQdG!O%z} zEPLqI7$cfcX^{afo℞Smim04*5Vy{4f>ZD#_PspzkJ#FJP&L$rv(1YY|Tq%Ec98 zqS<=^c%ATeBH0o1e=r?1$DqwKxwbi8&FS%bUv0{|wHr5HCI-Nyr4to_hxh1AvBy4p z-q{%vovS)43A#c-KSacdPjetREqx zmZJv;`Z$US4Tived>aOgj)1ZSI7|eA*c`wi#IJVHKq4}J2xYMR-SWuoxU?Z)?i zFoS_9rk*L?CX0#17$ZO%AsLuo%6G{^+j&~5zUjRvtWl3)@}*-}d+{&HMw(cw1 zr$p2&R8EW-<>yDiQK^PoM`qoE-C}@fIROU$dnRNL^-Hs>3+_E|U>9U4t;Jv6Xb!K`uX$cpGvT;YMtElUgEz(fr9s+ z(?Ju)72UsMas&Y6RU(thka3_H&_T+>S0fqnw>xX+o;|Hx0K8ct|F%F=JBA`mRP{5r zYQ`07PJd5u<>KZ(0LMJne1_9euL}}<(ITSp=S{j-!>l5;EzsT%K?s7hf)MSV!a+2P zY3?)(YZB%eSX8hWmoRjsr2t$cCl`*?_yeZjGQ65F&8*uGG>)m7by(Zp2Pgyh3XGMJ z+rCZ(*b&e3#G~pqNP`yrO@iu}{XqD}-I}l&yhz;<0NC{8*)v*(UB1g5m+rNY*P97n zhD$-FAz`qBOib)PDA$HSiy!OuByU&9E-fG?hja*dAOGfcYd;775P*!Y}94#DQ;(@PPA!fX3X5`e)0Rusak232LRg+&-i63&rf*c)0@d8{e|rW{dYB8z+vWu<3jEuBcX6t9h( zd89#7ejKweguYssjYfvRHVZ>bi_)^{n1mo64_*Fj2MtlyI&QVj+?j9^(bR zOax!HqGHlbe0>g+DT{4`TYnwCSq8_e!l{*yFp-lEVQdJsA78Pd2tSCpA`+&w1?T2* z{r;Kf_NrqX67tS~Dui@_!5ZdyB+?%Uz5fR0oqq=C;L9hjph^9?J3Sxs-ib{gWZ>LD z=7o}gFjzQv^5pu;%F6$zr7I7Maee=7DwJjvlD3%~I?B9aPeQBq2q zF|??hv?x+ZL}dvP#YE9|GKG}1ku6CmTm3$_@9z)Sb*{@X^UnJ|&vW1RXM1s*x&v>) z%7s8EMFcZ+z6H6Mivp1Ef$;?S=oS#Dd zHs1erEeqZJy_Fd6pr1%enuuS7GOMYtkNIN}UCXm7C5 z2#tQULi9^u{P>)fh%-gZMG0rYeVCk9J;V%3?j!iQ_h13~p>xhU59@8m*I@hg$$6gR z&uL{ELvc9COnO`hf_DbK@6W&IeSMGuDijr?80@P=a07SNSi9EIaS&Dgw4sP&ij0fo zro{Szw5WSHjY;iePol4VA1+lPCo2B`1tz%1=hGMN)kDQM+m@bJ*TsFKDgzg&%p&cb z@0!j@MVUMJ3FYS-pwq{%`r`+L6u@#VPDgOxYWGVKI4BVxRqqSVchD>So4(R<@QtWJ z=Se(^RRn$YMbgfp9y@%4Q%W-<_SCz@^Mr31e)(ucbV?`W$%7R*xw*o2; zt!VDt=T|+2=%-hud8U_rdAwZK-8W-T{2EQhu!OdiD{2a5>*T+N<8j-s;Q@RRsb=!* zGMAw}(%Fdih=^71a?kYTTsz~cb5Q#O%KAr}T0Nc>39z0z`O8}fj)95;kHLeQhQ5i| z`R10L*Z#qj6c&H1eyoA5zw{9jL!o1|wj$XZIuJ?xyKb0jw$qH^jq9M1p&5CIQ!~Ue z711Y@OJ2*mL7M`D7l8Y&xcTjftYflxLQ-Ooh!=pJ0PD>JbKOsiXI6JeLE1jtNp3kjn)fr<>GV?iuV*Y*RW zKVMS6s@7|C_8`KfQuDqKqiMR5ZzKJ6HjBlQxgjIq`Qng7B_s34JC*s~wb0Jdh!3^B zEGz54=m=F;G+2N~;4&cY-qdFr-#4bUuaJI^(OJ>kYb`r>YlV>;hJd(maMW?6V8cwt zihf7(Kk7wcy6Ex&G}((?V)jcLQHmEaDk*$jaaJ6cWys(`p1lt;X^flVV1hf~qwR?` zsyCBeD-UHbHRDjVax*ZzL%e}$zfvJ`@u~hvD&gjm80$?te7HBox3kgyQiF*@T2m7M z_pG9_Bp@S5^I%x~MYsV(A+teth4n=#EdC>*J5>v0X^n@Nv^;KX`#I0&+La1`GswLH z(%jq(Ewm+H&K2rz($BIk*oin;OAR>B!J-Z9Kx+!&>o^cP!w+|r~i_}-1 z6m}}h=4zq9z$tSBF!q- ztHW1CU@i$-dXdMK7QR+kF>q3njpMY88b<4jHF5})(z-&Y!;E#uQd6rb)&!!^uIspU z0y+W)t{|)Q66((a4kJpM`Vlo`IRIJFZ=v^R?`T&T^BQ08R)ueom$i(;B$OI5z|Bei&CXFa!D4wy>K zsPOS{CcwicOo!Zigv&y z_yC_`PeA%>EoCC1yL|>s+afxLQk4#j44%S_!HWNg`Ee^OR03fS&hkc3N#ZIA=8^!? z#>2Asb1pXj@Izd_@YKkcnTV-l9*O~+1K$mDO^T6x7A-pkZ5~A#p+rmDiWoN64~&Kp z3IUL*%i!qi-rfVtC5^aaRpxqKZQ<0l!fC94zbY*)m78iPHL&AmA&HSm0EjEJM!R7V z4Kvkq$_@(t`tIOxXr3n_ZX|x<#-fRC+W=+QpHBoUv_oyep|NtjUv5JmU%*DvNUF`m zl3lIGY0csu**m~ncG?4=A`PY((zN31aGigUH^&i|_3Lj$9)o)cnI$i967`wQ?wSeZ zFSUVxW#&22_9L)8r2X>@6;Hg1gJ)`kfUy!G_; zhx{ihXH3T#5_U*1S$LuX1vV9Zs&wuh7IBhkJOD`q7u2KEvVW0wEct zA>k5%lBsNj)Y1rx5O_}wo_f^N_b_UcHH4ItxTzDJpE3iC4vg$4TX$tqTHbd-&8rRf zm%8~R&Gl{z+G?O&HgyS4)g8kxAs=YASz(*dft2DXAkGrJ5xhKGAe07fMsapDC}n@d z1HVt+b0CAi9=Qk&tWk8uW+vWU3a7NX4_DQ@L#NUe@HUM%6vGGU4_*!PI6)r2a#DhqB7e&65bBa}y#V#j-Ah~7HYEOqx8hmmRb1Stt�-HD zWpcHHLnS=HN4SCbrQ1*tsOjJN%#6(1iWs0Ox{sRv5j$hHU{>`*(B+0lC(wE{5PW-U znal2kW3?Sh!`+3-(2ApDS2&*w zaCACOtpdn^K#22Rc%)mSt+@fX1Y2So&?;c$nw5!w$|4DFBx?k|z49=)Z;CLXY@_(< zrhwlY5UtvyG>3Z-%gTtw>fD)S(&Y$*2nBiznn23RdC}4rh>5nIU@+v8VN#<3qp}=Q zP9t~_UBbR^qb3EnLa`SaA#ekOCACro*x{dP*E>GLLi+4_gnk7(Kdlmz?sCMBpu9cA z*qMQ>89HQ1wPX1EddiOAkD`;72214uU2LN?0LsgUA&HKYA`1Xb^OlLYi=K(4v@( zJd|O2>cNyq@{oA13pKBP9A|81u%U99Cl(cZLUXknCSET#x4fDGSZ?fqL<*DCrrIr3=% zJNehmr35YC&oC(Czn>;hR)?{zs6D_qbH1Vl#{!ye4|NTVa(SF;L?KXf4qT+QmV{L^Kh(LO|~~a^!ve_T6?NEh99zn9?7J)l3RV`L~X z1-14DE~zLPg*>#U5Oh_ZnpfUnzIgqCSmsj8kW3x$QmT=un&eT^7D|I z{_dD5|CQ`;loOgoPsXPMSJ;OpfmsEgw!7cbr4N5cYsWEQXi$;aDZYUf|FAkyYae{S zjMKf?#e|~($V^1876ul;QO$6fk>i76260B&Q6@)BjAit-D)o?9ltD$`+HdK3Ee~J$ z?US`vVd>4C8=#4$SNris_XHY>WOw1{_>y)3Ssp>x z6Mv58`Cb(PW;=>*^SOcPdCu?l7*Ao|b_w(OA{3V^lh6=b17tFSWLQ-!@Ba|cRC z%gq2JfG@yGJ_HLP^#`8m(K;vUtT6#E0E9|LJVbHl0EH%d2}VC&kfH4{`VsgmR=hJD zY2f0yNsO~hY0R-Ic*g`n;Z?7Y?G1}#GM;H?9c-r1f_(eXB1F+bwSpi6Z4?iz1tK4i z+@lg%_#Qz76oLTZv^6#vnVo=|(#!xuD<~`s-?HxwY>$opemS}gxdj5_zaMihMv2}o zS|Sk1DuP-QhAS`vD9n{K?1k@TcThSINdo%<0bZShEDY%)7zi4AbAUP@C^&ZxToKkR z<^`bvQi-M`Ndw3iq+WPXaI(NJzXXs7Zfp~(-Jf(ISiBekThm|rASOcG)l4fEWJs8p zoMqEPECDibVmU!&-H1~W&}I#4TV(z;5vLbfYOc{^qYS`Xlsky7o`~R9&S^!iBFKVX zu)IQoT%ZX+jRNz9W6PrHpVNi!qh5ILo#c8?Gd5_^$@5rFeR-bk;rYh^yAh6!s7@XZ zL!|D#1lT+O#U4eRxV$Djkg~h4{B~)^Do0Pf-g;XLY>0v%%ik{WLYB@ zA#Xv)6c2R)E53$QXy9YuVG+Fp|)Sf+rcp z)eGU+-g@2_=&0x}cW)J@Zv<{X{;}C0$S6zeU2b-EB>X0ne};$0w8-}a3Q1iPS|fJ+ za!rhx?;tu*i)KX`fB?VN(CPCmJcbdl&e^#Rv=I*79Q;mMR+AwdXCKQ3?Wpy?VUn4W zGBnTik!CFBJ6!}`{zimcCRGwWCrT!zqa3I}bn!e9Ck^NZ@k0a+x80c|x77`EO@hHT z#sdJ1k?DEc;52EM;NhpkX-Py2&MY()(TpcOgWqqUloj6!uy`0yN-2!ve{QL(E?8pvjj$I>_-J7*7 z1dr!=2AZp02gtd2WOyqqL@P+&ExUty8k~XHg)+YP@@_d5ZZ8&-j6svS3ANH)(7g_D zC83zY&MXuLS=PN)OdNELFua;&6x-%`x&WyEwLUz;3h?W5pvPx0R8H2M=-UR~8UKU= zj409Q&kyWGIQwUc*@e53a$9EB!nfJ1udiRE9Ai+NfGJxNri&@cd@ccf-7)%iI^eTL zs0$)#gPtLTVSbyBBw}f7niNEw-B4+5FSaJ4gn%G z17Kk418U-m5EUeN@P6kP7KQ;644OE3>t^#fmAM-vzDPc1%!K-L5F8VkCI1(egSgD) z7~c>f&AXB?``wo@9~a_c6*kY2p4>AKB!T`oCVW~rmv{RafI@JpS?Kae;T

1yHw(>;H{7h#8^`y>nEBChs<|BM|e1NmE|}n>}F3_fS5z1w3^oY^LeS6ZX3l z&-9_2PK64W3?yTqz}$iU=W#@#Aed}Vi-$WT(QGfpO$K(S3VRzxM-W-psM!wC`GBxX zP6?T|*Vl`#-Z3-Ic7Zk++mW6C&T1T2ctwU#8<9Jw5290!wc!b}d zs$J1N&}6I{zcOM+1dn$cdPmBsfkEU5CV9k1Y$C0~@r~Y{h~>8Aq~1=?`>L<4t!*PN zedJTnqGZg(n^gF|ij2egdl6bo(TvEGqR?L)kqnEBR7sf$f2^qJ%!qoIf3U{c*(3{Z zmY9FUVAT?sEJtvwY%2o75u1%6tqai3AA{ZQm+Ghr7EN)u&j<&it^=lIwu*z8@;WD{ zGS!0+E%ycX@8POWsxae40dt=X@C4Ph5nLqVFqqI4pdR^>{T9O;b%F9VoH`k2y{8Mc zADEdVRPhAV5^g8zo~Siuft8{L3%>yAp)wFP1I~jIo6-$1U2Vt4K;z}?{CgR4_=A+( zvhop+YsGingX*G|Z9XQ-9JxV4Y(5@bh$Qx)^JN$*b<$ck@zkkMzI#xfXT6N~#S0fW z_6huZD1+JPnI^tD%=M((r5^HO&L6P!JhWl2VSL-#e15Iq*t?A)&QGij9i=m;Z=iE? z+dSsK$-a-3`FF8^ssLnpK&fYu{~AkbMFuyKNNRi+_Z(__i|CbKa<+jy-pjy+?<#Ck zLM-ayz@%P++{qYyeUV<`{?tprE6PpHdJrCce5^I8@irFghWEKfKh->)pW$wx){j zGW5I$fDEWpQtX?X@r=%TjxClb<+7k#vCc5W;H`&Lm7gjDlJdT(h_ASdgGsA`&)r6` zZ0cdduF?}!1&mI(mC+DZ!i8M{%!YCH-%X;crxAPx7rXqxEL3-`@V#PczRU0uhLgy~ z@oU+Za{9e7^spk%KM;pdY#~G;)){_i-L){>Effen1X{~%ZEwp2jb%{hYcH^b48p}meH|5`!nrxtZ6JJVSkqc z3V=Ie8^~Eqr!~`fg=qhOul!7oa@q2}4!F)yjZd@LxlmlE0y0QO!}S>5Yc#h7$P%@a zlzL0b;pmJ}U2W|km4OqS%6?pCsNC>Eb`MH~0cLng4S^vroFTspBi4#ji!+IshoD^6 zBt6drsY@C*s>yur41C>6$)jwGq&!a&Ob`eMK^sVRi>4GjK%klr;cgTH^8OH<_9M`a`-_t_vzFWj#^qA@UPtci|h~;~TA173J+` z;u{Z0o-~VI1Zf=@SuN|>4vffmv1Eo#u$D-f0Gs_~aH7y%F0KsAj4zARH)zd>SFBth+bif`5uI#MQjjL9D;$+)#Dt%!!x*wIE zXKN9i2~_a}p$fAKZ$zRejJ^pLYZz<|jY|l^v5Gz;1BK{Go2E2A*Bci)Bz8<(jTs0Y z7x3CESY5j)Z33i;?0ZzFzc3P^L7`wUj6~6AvakY-Gwc5rbZ4{dmS6?Q=Xu_df5voO z{xkr(h6liM9+7MfeKGS1I^JqzdDmlD`wk+CScKwr<70L3M5p++aVp1Bc*vymyihTo z*1eN>Iv<`Tf(a!hR3k?8`DOu<1jRU3>#8}sa`KRDj51E4C9M(IS(l%W7OdpI;mp$W={s1o}F-NHK+h>=w4*crYAZIUSL4JT{ zgJ?z=JLwalcRKbz1m=~SXL=y=WM*46&601_gvpVa^Zv(d5vPc!tqXY&aKFYop&{gN z2kyirNeTMh@YX(ed>mnDnVEHfCdq&aFh#Zm;bS;!rlCxnXMD9o9Rv|H6^GP)L@3^3 zbWK6QAvlnzXnIu%5MZjH*eiRStCryy0ZRJe=V)uxsPe=9oAKQZcx~X|i=Uw^AAxnczQXTuxk{=mIqW$H&u7XTc2 z@ZrPkSAJ;-#!8w#wmpP*vZ&chP0HKF#pNUW#JvRDDOCgg!TkziEVe%T2phvJ#Y~-9 zeR#{IC8t+qt)IPGTdBHtkPPWxl8_~F`RY~PZI`p_3hWD-^Pj$-zVu#On`-j1gMqi= zsy*KX_nSHuG>^!&IPiBK=EfI&ZGS1_#$2#GZ3Tb-;)da2Zg+QgkKcj^4<0B_o2Hdg za|!&2wT%s4!}Hg%hY1he1la=ZBSGtQlG$>WC!6~M-|jJ9RPe%EOS$9uKuX|n2-pJ-%TT#%wq1OG%(21655leZzo_~&}wj|?&Rbq%hHW3ewU^)CaIuiR@lnFAH9E;*<8P=57|f}9(<=Jc z7BnwR*u`YG=C$lppQ@oTy0N#dxdIFtUnt?Tt7Jncqh%iJ^2K%Q*6EZweGKa`)U@Wg z6|2p=a(!Jv^UQ=@li00=jUF8T(ze@ufeWYbJKy7j&Fq!bzO9q{nFJe_G1l|`qC5czWxYZ z(qNd|8y6RMzh&n)TrH0m+`9Kf-Rp(Z;vST1ndtI@T?(4Laz$@LL~pOkv`a||1YeF9 zriIRy6K>k%rxZFL!?JHp8@OSe4&~mb{cryz|(r1TU^> zGjR`O6>BgPclF#ZUnnQMo?P>4X5yVKOI{+|XjFY)N)Q^>2v*e|ro8}5TyE`uy>|MCZjOXS*TYSD_L9@PQ zXhT8-vua>W`%lv`-gwlie)Xa4wTm4JnvW+&Y_6_I%gESY;Qvwc6MkrYy^}J3J{p<> zlcs9(<#H`fC-m(xzHVjzrPWLzs7`wesA{E+{g+0sk{=q?6?5FO?)(1o>i=!tfs4sF zc|dCMs*CGJ$4*_(mn-AOOLl~||2!(*tHu6&Z(WA}b>rT}w~LGavyx6{S>`zCysGE+(r*o8-66q^6dKAE?(}}B~vun!jGG3KS(Z3`Xy=TAUAcH zk&)5eckkZGTY|q<)zjXWl7r(p%{EM%l2bi>tn)WW#l9`!D92aRRxUfd$Ps zwT@)BdX+raF}y05IJ$cF8Nr-&b|9#m$VTEHrm zqa8fulMOc9WBu2e{@3GTW82o~r7RSbiCOkotyr-lH%jwHW3Bg?BZ4JG#_CdcHDt^` zrGcND$2R4&{L*L4m|=L%;oXe>-~E#Nc)5L^#XUn&TN41 zDMw>t%gRM>%cgNPnEU3`X&?EBW2`*KrO5Pf^^7^ZYi77iiGHkV_l{&i^=LqC?c!G% zw{EQ~xX8ZzP%crIgNwLu;X>KeVYLn&O`cm&K!l}x;C7wlAInOe0tGvnmTo61eHV3R z{FFBe(Jxj=?6epEh-H1vrO1)dBF(R!cjDy93G?R7YnR*4xG`=C%P`2ef@SnGW?eym zho|ScnCf|kyx?k5J=B+O@~^)wb3-|C@pku@<0CutZ@u^!+U!-rUvkud7wn)~Wva&S zj6>_JR-w-OQ6<9jDMCYZyV>ijJuf(c^|%|gWAzK~q2AgL(*3^!%qAqZ`#-9^Bekgb z(xrj<D&(t2iYgL5N_BotCxo+eFSaCnJ^Ay3-u()txAS|*t@occ3vO?Vbnjre`p z$ntOPrf%+=iX3!wc|S7nV=rL$sPa3T>q~cpKNS~epKTC7y00Hc-%}JFg@*ciW1h5t zW!SOo%ww+4TpxL#zkFEi(U(bSp*10+b#`a{JzQN+$5)FP@%S;x%g)4P`0pMY*|Dr+ zoMwz%PC!Sefn0gx+dX$m5OG)KzJ0qI+wvY`>K24=Yt@%7&-CAoC;PQxTDWtyrt#QC z68l?gNA%=`LY%p~c>I6yO|t+u+AIBDq0CQN*jxCiebYFlgfIu7E3U&Y@B%`Be7jt@nMMq`qC>aB)7*{>z4rk_x@tlng2Lm1kBT-D8|7G-s%ubI@7IVw)zh z*xRT7`QY2CH=fnHmkkw3yGZ_s#x-_*`9NF z6x=^EVyRKgtry;SF4Gy3iWT0P3b}9e)i#e^FBLDj^m^;XXxG%B{#owp#FKDT@p#t` zve+iOfbCx^F7{bmEb~l0R`OQa&@CK5g$_C;H6i6SwGYaMaw;ZlmMNB09%CKBX?Q3m zEoD~e|2$lBQE|Qhyzsf3<+)=rB<#{!(ueN2y3fkA+y0lFaFWr+fp#ptv74FepCJ>$ j$R=KOK(M6Y$I7MV!P$w+&&`_). + +Notice we mention fingerprints, and not the source code itself. Keeping the privacy of your information is the most important rule we follow, +and what makes us different than our competitors. In order to achieve this, the SCANOSS Platform calculates file and snippet fingerprints +(32-bit identifiers calculated with the `winnowing algorithm `_). + +The fingerprints of each file or snippet are then sent to the `SCANOSS API `_, that means you are scanning against the knowledge base and not the other +way around. + +One way to query the SCANOSS Platform is through our Python package: `scanoss-py `_. + +.. note:: + All of SCANOSS software is open source and free to use, explore our `GitHub Organization page `_. You can contribute to this tool, for more information check the `contribution guidelines `_ for this project. + +Features +======== +* The package can be run from the command line, or consumed from another Python script +* Scan your source code fingerprints against a knowledge base +* Dependency detection +* Decoration services for cryptographic algorithm, vulnerabilities, semgrep issues/findings and component version detection +* Generate an SBOM (software bill of materials) in SPDX-Lite and CycloneDX + +Installation +============ +To install (from `pypi.org `_), run: ``pip3 install scanoss``. + +------------ +Requirements +------------ + +Python 3.7 or higher. + +The dependencies can be found in the `requirements.txt `_ and `requirements-dev.txt `_ files. + +To install dependencies run: ``pip3 install -r requirements.txt`` and ``pip3 install requirements-dev.txt``. + +To enable dependency scanning, an extra tool is required: scancode-toolkit. + +To install it run: ``pip3 install -r requirements-scancode.txt`` + + + +Commands and arguments +====================== + + +------------------ +Scanning: scan, sc +------------------ + +Scans a directory or file (source code or ``.wfp`` fingerprint file) and shows results on the STDOUT, by default. This command is highly customizable, from the output format to the matching selection logic using an SBOM file, everything can be set to your preference. + +.. code-block:: bash + + scanoss-py scan + + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --wfp , -w + - Allows to scan a wfp (winnowing fingerprint) file instead of a directory + * - --dep , -p + - Use a dependency file instead of a directory + * - --identify , -i + - Scan and identify components in SBOM file (an API key is required for this feature) + * - --ignore , -n + - Ignore components specified in the IGNORE SBOM file (an API key is required for this feature) + * - --format , -f + - Indicates the result output format: {plain, cyclonedx, spdxlite, csv} (optional - default plain) + * - --flags , -F + - Sends scanning flags (or definitions) + * - --threads , -T + - Number of threads to use while scanning (optional - default 10 - max 30) + * - --skip-snippets, -S + - Skip the generation of snippets + * - --post-size , -P + - Number of kilobytes to limit the post to while scanning (optional - default 64) + * - --timeout , -M + - Timeout (in seconds) for API communication (optional - default 120) + * - --no-wfp-output + - Skip WFP file generation + * - --all folders + - Scan all folders + * - --all-extensions + - Scan all file extensions + * - --all-hidden + - Scan all hidden file/folders + * - --obfuscate + - Obfuscate fingerprints + * - --dependencies, -D + - Add dependency scanning + * - --dependencies-only + - Run dependency scanning only + * - --sc-command + - Scancode command and path if required (optional - default scancode) + * - --sc-timeout + - Timeout (in seconds) for Scancode to complete (optional - default 600) + * - --apiurl + - SCANOSS API URL (optional - default https://api.osskb.org/api/scan/direct) + * - --ignore-cert-errors + - Ignore certificate errors + * - --key , -k + - SCANOSS API Key token (optional - not required for default API_URL) + * - --proxy + - Proxy URL to use for connections, can also use the environment variable ``HTTPS_PROXY`` (optional) + * - --pac + - Proxy auto configuration (optional). + * - --ca-cert + - Alternative certificate PEM file, can also use the environment variables ``REQUEST_CA_BUNDLE`` and ``GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`` (optional) + * - --api2url + - SCANOSS gRPC API 2.0 URL (optional - default https://api.osskb.org/api/scan/direct) + * - --grpc-proxy + - GRPC Proxy URL to use for connections, can also us the environment variable ``GRPC_PROXY`` (optional) + +------------------------------------------- +Generate fingerprints: fingerprint, fp, wfp +------------------------------------------- + +Calculates hashes for a directory or file and shows them on the STDOUT. + +.. code-block:: bash + + scanoss-py fingerprint + + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --output , -o + - Output result file name (optional - default STDOUT) + * - --obfuscate + - Obfuscate fingerprints + * - --skip-snippets, -S + - Skip the generation of snippets + * - --all-extensions + - Fingerprint all file extensions + * - --all-folders + - Fingerprint all folders + * - --all-hidden + - Fingerprint all hidden files/folders + +----------------------------------------- +Detect dependecies: dependencies, dp, dep +----------------------------------------- + +Scan source code for dependencies, but do not decorate them. + +.. code-block:: bash + + scanoss-py dependencies <> + + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --output , -o + - Output result file name (optional - default STDOUT) + * - --sc-command SC_COMMAND + - Scancode command and path if required (optional - default scancode) + * - --sc-timeout SC_TIMEOUT + - Timeout (in seconds) for scancode to complete (optional - default 600) + +.. note:: + Remember that in order to enable dependency scanning, an extra tool is required: **scancode-toolkit**. To install it, run: ``pip3 install -r requirements-scancode.txt``. + +-------------------------- +File count: file_count, fc +-------------------------- + +Search the source tree and produce a file type summary. + +.. code-block:: bash + + scanoss-py file_count + + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --output , -o + - Output result file name (optional - default STDOUT) + * - --all-hidden + - Scan all hidden files/directories + +----------------------------------------- +Format conversion: convert, cv, cnv, cvrt +----------------------------------------- + +Convert file format to plain, SPDX-Lite, CycloneDX or csv. + +.. code-block:: bash + + scanoss-py convert -i --format -o + + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - -input , -i + - Input file name. + * - --output , -o + - Output result file name (optional - default STDOUT) + * - --format , -f + - Indicates the result output format: {plain, cyclonedx, spdxlite, csv}. (optional - default plain) + +----------------- +Component: +----------------- + +To be done + +------------------------ +Utilities: utilities, ut +------------------------ + +.. code-block:: bash + + scanoss-py utilities + + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - fast + - SCANOSS fast winnowing (requires the `SCANOSS Winnowing Python Package `_) + * - certloc, cl + - Display the location of Python CA certificates + * - cert-download, cdl, cert-dl + - Download the specified server's SSL PEM certificate + * - pac-proxy, pac + - Use Proxy Auto-Config to determine proxy configuration + +----------------- +General Arguments +----------------- + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - -debug, -d + - Enable debug messages + * - --trace, -t + - Enable trace messages, including API posts + * - --quiet, -q + - Enable quiet mode + + +Package consumption +==================== + +This package can be run from the command line, or consumed from another Python script. A good example of how to consume it can be found in this `file `_. + + +In general the easiest way is to import the required module as follows: + +.. code-block:: python + + from scanoss.scanner import Scanner + + def main(): + scanner = Scanner() + scanner.scan_folder( '.' ) + + if __name__ == "__main__": + main() + + +Alternatively, there is a docker image of the compiled package, which can be found in this `repository `_. Details on how to run it can be found in this `file `_. + + +Integrations +============ + +At SCANOSS we want to provide **easy recipes for practical solutions**, that is the reason we are constantly working on building integrations for our software. No need to adapt your existing systems to work with our software, we will adapt our software to your needs. + + +From CI/CD integrations with `Jenkins `_ and `GitHub Actions `_, to our `SonarQube plugin `_ and our most recent `VSCode extension `_. We are always working on making our software as easy to access, consume and integrate as possible. + + +The full list of existing integrations is down below: + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Integration + - Description + * - `Jenkins `_ + - Integrate scanoss-py into your pipelines + * - `GitHub Actions `_ + - Enhance your software development process with the SCANOSS Code Scan Action + * - `SonarQube `_ + - Scan your code with the SCANOSS plugin for SonarQube + * - `Visual Studio Code `_ + - Software Composition Analysis as you code + +Best practices +============== + +| + +---------------------------------------------------------------------- +*Choose the tool based on your use case, and not the other way around* +---------------------------------------------------------------------- + +SCANOSS offers many tools and software in the field of Software Composition Analysis, and many have similar features. + + +For example, you can perform scans and generate software bill of materials (SBOM) with scanoss-py and the `SBOM Workbench `_, but that doesn't mean these tools are interchangeable. The SBOM Workbench's GUI can be an advantage for auditors and such, but may be a complication for developers that need to integrate an SCA solution into an existing workflow. + + +There is also the case for language preferences, we also offer a `Javascript package `_ and a `Java SDK `_ so you have freedom to consume the SCANOSS API however you want. + +| + +------------------------------- +*Get the most accurate results* +------------------------------- + + +License +======= +The Scanoss Open Source scanoss-py package is released under the MIT license. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: \ No newline at end of file From f5c0c3516ca0dc426f9c5411722e9632486ae362 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Wed, 14 Aug 2024 14:27:21 -0300 Subject: [PATCH 144/489] added logo to user guide --- docs/source/conf.py | 2 +- docs/source/{_static => }/scanosslogo.png | Bin 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/source/{_static => }/scanosslogo.png (100%) diff --git a/docs/source/conf.py b/docs/source/conf.py index 00373145..905131dc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,5 +24,5 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = 'furo' -html_logo = 'static/scanosslogo.png' +html_logo = 'scanosslogo.png' html_static_path = ['_static'] diff --git a/docs/source/_static/scanosslogo.png b/docs/source/scanosslogo.png similarity index 100% rename from docs/source/_static/scanosslogo.png rename to docs/source/scanosslogo.png From c48d10ef6528239f239b84a0fb78869d644df6a9 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Thu, 22 Aug 2024 21:58:52 -0300 Subject: [PATCH 145/489] update --- analyzer-result.yml | 5027 +++++++++++++++++++++++++++++++++++++++++ docs/source/conf.py | 2 +- docs/source/index.rst | 7 +- 3 files changed, 5029 insertions(+), 7 deletions(-) create mode 100644 analyzer-result.yml diff --git a/analyzer-result.yml b/analyzer-result.yml new file mode 100644 index 00000000..46e26d4f --- /dev/null +++ b/analyzer-result.yml @@ -0,0 +1,5027 @@ +--- +repository: + vcs: + type: "Git" + url: "https://github.com/scanoss/scanoss.py" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/scanoss/scanoss.py.git" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "" + config: {} +analyzer: + start_time: "2024-08-17T19:07:21.548502047Z" + end_time: "2024-08-17T19:08:17.490737378Z" + environment: + ort_version: "DOCKER-SNAPSHOT" + build_jdk: "11.0.24+8" + java_version: "17.0.12" + os: "Linux" + processors: 10 + max_memory: 2055208960 + variables: + HOME: "/home/ort" + JAVA_HOME: "/opt/java/openjdk" + ANDROID_HOME: "/opt/android-sdk" + tool_versions: + NPM: "10.7.0" + config: + allow_dynamic_versions: false + skip_excluded: false + result: + projects: + - id: "NPM::tests/data/package.json:" + definition_file_path: "tests/data/package.json" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/scanoss/scanoss.py.git" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "tests/data" + homepage_url: "" + scope_names: [] + - id: "PIP::data:f5c0c3516ca0dc426f9c5411722e9632486ae362" + definition_file_path: "tests/data/requirements.txt" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/scanoss/scanoss.py.git" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "tests/data" + homepage_url: "" + scope_names: + - "install" + - id: "PIP::docs-docs:f5c0c3516ca0dc426f9c5411722e9632486ae362" + definition_file_path: "docs/requirements-docs.txt" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/scanoss/scanoss.py.git" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "docs" + homepage_url: "" + scope_names: + - "install" + - id: "PIP::requirements-dev.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362" + definition_file_path: "requirements-dev.txt" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/scanoss/scanoss.py.git" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "" + homepage_url: "" + scope_names: + - "install" + - id: "PIP::requirements-scancode.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362" + definition_file_path: "requirements-scancode.txt" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/scanoss/scanoss.py.git" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "" + homepage_url: "" + scope_names: + - "install" + - id: "PIP::requirements.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362" + definition_file_path: "requirements.txt" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/scanoss/scanoss.py.git" + revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" + path: "" + homepage_url: "" + scope_names: + - "install" + packages: + - id: "PyPI::alabaster:1.0.0" + purl: "pkg:pypi/alabaster@1.0.0" + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "A light, configurable Sphinx theme" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl" + hash: + value: "fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz" + hash: + value: "c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/sphinx-doc/alabaster.git" + revision: "" + path: "" + - id: "PyPI::attrs:24.2.0" + purl: "pkg:pypi/attrs@24.2.0" + authors: + - "Hynek Schlawack " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Classes Without Boilerplate" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/6a/21/5b6702a7f963e95456c0de2d495f67bf5fd62840ac655dc451586d23d39a/attrs-24.2.0-py3-none-any.whl" + hash: + value: "81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/fc/0f/aafca9af9315aee06a89ffde799a10a582fe8de76c563ee80bbcdc08b3fb/attrs-24.2.0.tar.gz" + hash: + value: "5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::babel:2.16.0" + purl: "pkg:pypi/babel@2.16.0" + authors: + - "Armin Ronacher " + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "Internationalization utilities" + homepage_url: "https://babel.pocoo.org/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ed/20/bc79bc575ba2e2a7f70e8a1155618bb1301eaa5132a8271373a6903f73f8/babel-2.16.0-py3-none-any.whl" + hash: + value: "368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/2a/74/f1bc80f23eeba13393b7222b11d95ca3af2c1e28edca18af487137eefed9/babel-2.16.0.tar.gz" + hash: + value: "d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/python-babel/babel.git" + revision: "" + path: "" + - id: "PyPI::backports-tarfile:1.2.0" + purl: "pkg:pypi/backports-tarfile@1.2.0" + authors: + - "\"Jason R. Coombs\" " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Backport of CPython tarfile module" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl" + hash: + value: "77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz" + hash: + value: "d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::banal:1.0.6" + purl: "pkg:pypi/banal@1.0.6" + authors: + - "Friedrich Lindenberg " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Commons of banal micro-functions for Python." + homepage_url: "http://github.com/pudo/banal" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ae/c4/7f6e6a539cc6b2da4da3b6a58d5e6f9342c870522ee46d41f8cbd2156953/banal-1.0.6-py2.py3-none-any.whl" + hash: + value: "877aacb16b17f8fa4fd29a7c44515c5a23dc1a7b26078bc41dd34829117d85e1" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/8c/a7/3301a69b4a31f7324b99332d758ae8da691f7f865ccd1b2adcd973c45344/banal-1.0.6.tar.gz" + hash: + value: "2fe02c9305f53168441948f4a03dfbfa2eacc73db30db4a93309083cb0e250a5" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pudo/banal.git" + revision: "" + path: "" + - id: "PyPI::beartype:0.18.5" + purl: "pkg:pypi/beartype@0.18.5" + authors: + - "Cecil Curry, et al. " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Unbearably fast runtime type checking in pure Python." + homepage_url: "https://beartype.readthedocs.io" + binary_artifact: + url: "https://files.pythonhosted.org/packages/64/43/7a1259741bd989723272ac7d381a43be932422abcff09a1d9f7ba212cb74/beartype-0.18.5-py3-none-any.whl" + hash: + value: "5301a14f2a9a5540fe47ec6d34d758e9cd8331d36c4760fc7a5499ab86310089" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/96/15/4e623478a9628ad4cee2391f19aba0b16c1dd6fedcb2a399f0928097b597/beartype-0.18.5.tar.gz" + hash: + value: "264ddc2f1da9ec94ff639141fbe33d22e12a9f75aa863b83b7046ffff1381927" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/beartype/beartype.git" + revision: "" + path: "" + - id: "PyPI::beautifulsoup4:4.12.3" + purl: "pkg:pypi/beautifulsoup4@4.12.3" + authors: + - "Leonard Richardson " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Screen-scraping library" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/b1/fe/e8c672695b37eecc5cbf43e1d0638d88d66ba3a44c4d321c796f4e59167f/beautifulsoup4-4.12.3-py3-none-any.whl" + hash: + value: "b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/b3/ca/824b1195773ce6166d388573fc106ce56d4a805bd7427b624e063596ec58/beautifulsoup4-4.12.3.tar.gz" + hash: + value: "74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::binaryornot:0.4.4" + purl: "pkg:pypi/binaryornot@0.4.4" + authors: + - "Audrey Roy Greenfeld " + declared_licenses: + - "BSD" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "Ultra-lightweight pure Python package to check if a file is binary\ + \ or text." + homepage_url: "https://github.com/audreyr/binaryornot" + binary_artifact: + url: "https://files.pythonhosted.org/packages/24/7e/f7b6f453e6481d1e233540262ccbfcf89adcd43606f44a028d7f5fae5eb2/binaryornot-0.4.4-py2.py3-none-any.whl" + hash: + value: "b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/a7/fe/7ebfec74d49f97fc55cd38240c7a7d08134002b1e14be8c3897c0dd5e49b/binaryornot-0.4.4.tar.gz" + hash: + value: "359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/audreyr/binaryornot.git" + revision: "" + path: "" + - id: "PyPI::boolean-py:4.0" + purl: "pkg:pypi/boolean-py@4.0" + authors: + - "Sebastian Kraemer " + declared_licenses: + - "BSD-2-Clause" + declared_licenses_processed: + spdx_expression: "BSD-2-Clause" + description: "Define boolean algebras, create and parse boolean expressions\ + \ and create custom boolean DSL." + homepage_url: "https://github.com/bastikr/boolean.py" + binary_artifact: + url: "https://files.pythonhosted.org/packages/3f/02/6389ef0529af6da0b913374dedb9bbde8eabfe45767ceec38cc37801b0bd/boolean.py-4.0-py3-none-any.whl" + hash: + value: "2876f2051d7d6394a531d82dc6eb407faa0b01a0a0b3083817ccd7323b8d96bd" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/a2/d9/b6e56a303d221fc0bdff2c775e4eef7fedd58194aa5a96fa89fb71634cc9/boolean.py-4.0.tar.gz" + hash: + value: "17b9a181630e43dde1851d42bef546d616d5d9b4480357514597e78b203d06e4" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/bastikr/boolean.py.git" + revision: "" + path: "" + - id: "PyPI::build:1.2.1" + purl: "pkg:pypi/build@1.2.1" + authors: + - "Filipe Laíns , Bernát Gábor , layday\ + \ , Henry Schreiner " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "# build" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl" + hash: + value: "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/ce/9e/2d725d2f7729c6e79ca62aeb926492abbc06e25910dd30139d60a68bcb19/build-1.2.1.tar.gz" + hash: + value: "526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::cachetools:5.4.0" + purl: "pkg:pypi/cachetools@5.4.0" + authors: + - "Thomas Kemmer " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Extensible memoizing collections and decorators" + homepage_url: "https://github.com/tkem/cachetools/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/04/e6/a1551acbaa06f3e48b311329828a34bc9c51a8cfaecdeb4d03c329a1ef85/cachetools-5.4.0-py3-none-any.whl" + hash: + value: "3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/a7/3f/ea907ec6d15f68ea7f381546ba58adcb298417a59f01a2962cb5e486489f/cachetools-5.4.0.tar.gz" + hash: + value: "b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/tkem/cachetools.git" + revision: "" + path: "" + - id: "PyPI::certifi:2024.7.4" + purl: "pkg:pypi/certifi@2024.7.4" + authors: + - "Kenneth Reitz " + declared_licenses: + - "MPL-2.0" + - "Mozilla Public License 2.0 (MPL 2.0)" + declared_licenses_processed: + spdx_expression: "MPL-2.0" + mapped: + Mozilla Public License 2.0 (MPL 2.0): "MPL-2.0" + description: "Python package for providing Mozilla's CA Bundle." + homepage_url: "https://github.com/certifi/python-certifi" + binary_artifact: + url: "https://files.pythonhosted.org/packages/1c/d5/c84e1a17bf61d4df64ca866a1c9a913874b4e9bdc131ec689a0ad013fb36/certifi-2024.7.4-py3-none-any.whl" + hash: + value: "c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/c2/02/a95f2b11e207f68bc64d7aae9666fed2e2b3f307748d5123dffb72a1bbea/certifi-2024.7.4.tar.gz" + hash: + value: "5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/certifi/python-certifi.git" + revision: "" + path: "" + - id: "PyPI::cffi:1.17.0" + purl: "pkg:pypi/cffi@1.17.0" + authors: + - "Armin Rigo, Maciej Fijalkowski " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "CFFI" + homepage_url: "http://cffi.readthedocs.org" + binary_artifact: + url: "https://files.pythonhosted.org/packages/f3/b9/f163bb3fa4fbc636ee1f2a6a4598c096cdef279823ddfaa5734e556dd206/cffi-1.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/1e/bf/82c351342972702867359cfeba5693927efe0a8dd568165490144f554b18/cffi-1.17.0.tar.gz" + hash: + value: "f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/python-cffi/cffi.git" + revision: "" + path: "" + - id: "PyPI::chardet:5.2.0" + purl: "pkg:pypi/chardet@5.2.0" + authors: + - "Mark Pilgrim " + declared_licenses: + - "GNU Lesser General Public License v2 or later (LGPLv2+)" + - "LGPL" + declared_licenses_processed: + spdx_expression: "LGPL-2.0-or-later" + mapped: + GNU Lesser General Public License v2 or later (LGPLv2+): "LGPL-2.0-or-later" + LGPL: "LGPL-2.0-or-later" + description: "Universal encoding detector for Python 3" + homepage_url: "https://github.com/chardet/chardet" + binary_artifact: + url: "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl" + hash: + value: "e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz" + hash: + value: "1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/chardet/chardet.git" + revision: "" + path: "" + - id: "PyPI::charset-normalizer:3.3.2" + purl: "pkg:pypi/charset-normalizer@3.3.2" + authors: + - "Ahmed TAHRI " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "The Real First Universal Charset Detector. Open, modern and actively\ + \ maintained alternative to Chardet." + homepage_url: "https://github.com/Ousret/charset_normalizer" + binary_artifact: + url: "https://files.pythonhosted.org/packages/40/26/f35951c45070edc957ba40a5b1db3cf60a9dbb1b350c2d5bef03e01e61de/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz" + hash: + value: "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Ousret/charset_normalizer.git" + revision: "" + path: "" + - id: "PyPI::click:8.1.7" + purl: "pkg:pypi/click@8.1.7" + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "Composable command line interface toolkit" + homepage_url: "https://palletsprojects.com/p/click/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl" + hash: + value: "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz" + hash: + value: "ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pallets/click.git" + revision: "" + path: "" + - id: "PyPI::colorama:0.4.6" + purl: "pkg:pypi/colorama@0.4.6" + authors: + - "Jonathan Hartley " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "Cross-platform colored terminal text." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl" + hash: + value: "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz" + hash: + value: "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::commoncode:31.2.1" + purl: "pkg:pypi/commoncode@31.2.1" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + description: "Set of common utilities, originally split from ScanCode" + homepage_url: "https://github.com/nexB/commoncode" + binary_artifact: + url: "https://files.pythonhosted.org/packages/57/ab/3c3f9117bf1d0131f43e192ad336c1b821efef6551156b1d64e70535e6c0/commoncode-31.2.1-py3-none-any.whl" + hash: + value: "c1ab57f014bf92b609f95b86e5ae5961afbd7cc83cd42c2a4b9bdb3b8453fa5e" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/e8/e0/7f942e4f9e52b8c3e665707f434a96b55d276aa0d8bea4aa4dc36ad4960b/commoncode-31.2.1.tar.gz" + hash: + value: "907a75e6a64e16e19c4072c80e2406d89bde3dcebf79963d7ec6578eca22a883" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/commoncode.git" + revision: "" + path: "" + - id: "PyPI::container-inspector:33.0.0" + purl: "pkg:pypi/container-inspector@33.0.0" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + description: "Docker, containers, rootfs and virtual machine related software\ + \ composition analysis (SCA) utilities." + homepage_url: "https://github.com/nexB/container-inspector" + binary_artifact: + url: "https://files.pythonhosted.org/packages/74/aa/966e81912c7771269e6608478521a40a16460868a02fa34059b1f8be0737/container_inspector-33.0.0-py3-none-any.whl" + hash: + value: "6284ac158c7115672ab70d1b97a22b6d257b59ee12bebc76c6048585943e919f" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/57/fe/50dd7a684f740bd66ab2969888723fe6c16acff41629fb731e7e131d78bf/container_inspector-33.0.0.tar.gz" + hash: + value: "09260edb14549648da61260c1559b507e9dcb8296a6324368ba3803ca2011f7c" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/container-inspector.git" + revision: "" + path: "" + - id: "PyPI::crc32c:2.6" + purl: "pkg:pypi/crc32c@2.6" + authors: + - "The ICRAR DIA Team " + declared_licenses: + - "GNU Lesser General Public License v2 or later (LGPLv2+)" + - "LGPL-2.1-or-later" + declared_licenses_processed: + spdx_expression: "LGPL-2.0-or-later AND LGPL-2.1-or-later" + mapped: + GNU Lesser General Public License v2 or later (LGPLv2+): "LGPL-2.0-or-later" + description: "A python package implementing the crc32c algorithm in hardware\ + \ and software" + homepage_url: "https://github.com/ICRAR/crc32c" + binary_artifact: + url: "https://files.pythonhosted.org/packages/bd/95/edb7fd426f2a092e4d36377814c47d095c642710c55b1754679294bbc220/crc32c-2.6-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "eb867368bcd541933dd117074f836ce59f90ebac57df06dc1194ae669ad8f6fc" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/0d/49/d51f2ad63f7afc4a3c426c725c298ec6f74c7c00ce2609fc717fea0c533d/crc32c-2.6.tar.gz" + hash: + value: "f8c0d09e168c8af4c98fe61c772c775a2ec5d5bcc7a57f095daed423730309c8" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/ICRAR/crc32c.git" + revision: "" + path: "" + - id: "PyPI::cryptography:43.0.0" + purl: "pkg:pypi/cryptography@43.0.0" + authors: + - "The cryptography developers >" + declared_licenses: + - "Apache Software License" + - "Apache-2.0 OR BSD-3-Clause" + - "BSD License" + declared_licenses_processed: + spdx_expression: "Apache-2.0 AND (Apache-2.0 OR BSD-3-Clause)" + mapped: + Apache Software License: "Apache-2.0" + BSD License: "BSD-3-Clause" + description: "cryptography is a package which provides cryptographic recipes\ + \ and primitives to Python developers." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/77/9d/0b98c73cebfd41e4fb0439fe9ce08022e8d059f51caa7afc8934fc1edcd9/cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/69/ec/9fb9dcf4f91f0e5e76de597256c43eedefd8423aa59be95c70c4c3db426a/cryptography-43.0.0.tar.gz" + hash: + value: "b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::debian-inspector:31.1.0" + purl: "pkg:pypi/debian-inspector@31.1.0" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0 AND BSD-3-Clause AND MIT" + declared_licenses_processed: + spdx_expression: "Apache-2.0 AND BSD-3-Clause AND MIT" + description: "Utilities to parse Debian package, copyright and control files." + homepage_url: "https://github.com/nexB/debian-inspector" + binary_artifact: + url: "https://files.pythonhosted.org/packages/09/61/709907d112553b39c8c907f0f90618de58ec87ca4565959d2ad350c84b9f/debian_inspector-31.1.0-py3-none-any.whl" + hash: + value: "77dfeb34492dd49d8593d4f7146ffa3f71fca703737824e09d7472e0eafca567" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/90/7a/6f9d38aabf50c1e0449e22e42485047f9d22792664e1006b14aba8d2f604/debian_inspector-31.1.0.tar.gz" + hash: + value: "ebcfbc17064f10bd3b6d2122cdbc97b71a494af0ebbafaf9a8ceadfe8b164f99" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/debian-inspector.git" + revision: "" + path: "" + - id: "PyPI::dockerfile-parse:2.0.1" + purl: "pkg:pypi/dockerfile-parse@2.0.1" + authors: + - "Jiri Popelka " + declared_licenses: + - "BSD" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "Python library for Dockerfile manipulation" + homepage_url: "https://github.com/containerbuildsystem/dockerfile-parse" + binary_artifact: + url: "https://files.pythonhosted.org/packages/7a/6c/79cd5bc1b880d8c1a9a5550aa8dacd57353fa3bb2457227e1fb47383eb49/dockerfile_parse-2.0.1-py2.py3-none-any.whl" + hash: + value: "bdffd126d2eb26acf1066acb54cb2e336682e1d72b974a40894fac76a4df17f6" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/92/df/929ee0b5d2c8bd8d713c45e71b94ab57c7e11e322130724d54f469b2cd48/dockerfile-parse-2.0.1.tar.gz" + hash: + value: "3184ccdc513221983e503ac00e1aa504a2aa8f84e5de673c46b0b6eee99ec7bc" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/containerbuildsystem/dockerfile-parse.git" + revision: "" + path: "" + - id: "PyPI::docutils:0.21.2" + purl: "pkg:pypi/docutils@0.21.2" + authors: + - "David Goodger " + declared_licenses: + - "BSD License" + - "GNU General Public License (GPL)" + - "Public Domain" + - "Python Software Foundation License" + declared_licenses_processed: + spdx_expression: "GPL-3.0-or-later AND LicenseRef-scancode-public-domain-disclaimer\ + \ AND PSF-2.0" + mapped: + GNU General Public License (GPL): "GPL-3.0-or-later" + Public Domain: "LicenseRef-scancode-public-domain-disclaimer" + Python Software Foundation License: "PSF-2.0" + unmapped: + - "BSD License" + description: "Docutils -- Python Documentation Utilities" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl" + hash: + value: "dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz" + hash: + value: "3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::dparse2:0.7.0" + purl: "pkg:pypi/dparse2@0.7.0" + authors: + - "originally from Jannis Gebauer, maintained by AboutCode.org " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A parser for Python dependency files" + homepage_url: "https://github.com/nexB/dparse2" + binary_artifact: + url: "https://files.pythonhosted.org/packages/22/e9/a370e566f84807cff908e71a4824ae00ea8196319f4e2956e82509a5f1c6/dparse2-0.7.0-py3-none-any.whl" + hash: + value: "2b935161700cdad4f27fa7ada85900756739be65ba3ef614ac4436e7ba929102" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/dc/d2/59a42c7b40c1075d49aa6b5ea32a5baa87f8022d252ccb4762ca9d5a30f5/dparse2-0.7.0.tar.gz" + hash: + value: "6bf6872aeaffedcac67ad0abb516630bad045dbdb58505b58d8f796ee91f0a73" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/dparse2.git" + revision: "" + path: "" + - id: "PyPI::dukpy:0.4.0" + purl: "pkg:pypi/dukpy@0.4.0" + authors: + - "Alessandro Molina " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Simple JavaScript interpreter for Python" + homepage_url: "https://github.com/amol-/dukpy" + binary_artifact: + url: "https://files.pythonhosted.org/packages/b2/11/ef428e024465396d8c76a7c6ea9047bf95c33ed7070bd45e96fab164c704/dukpy-0.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "601adc77605fa83ad6f4b201fd6701528c1eee1ec7de2465ca8a23c636f39552" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d1/0b/402194ebcd92bb5a743106c0f4af8cf6fc75bcfeb441b90290accb197745/dukpy-0.4.0.tar.gz" + hash: + value: "677ec7102d1c1c511f7ef918078e8099778dbcea7caf3d6a2a2a72f72aa2d135" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/amol-/dukpy.git" + revision: "" + path: "" + - id: "PyPI::fasteners:0.19" + purl: "pkg:pypi/fasteners@0.19" + authors: + - "Joshua Harlow" + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "A python package that provides useful locks" + homepage_url: "https://github.com/harlowja/fasteners" + binary_artifact: + url: "https://files.pythonhosted.org/packages/61/bf/fd60001b3abc5222d8eaa4a204cd8c0ae78e75adc688f33ce4bf25b7fafa/fasteners-0.19-py3-none-any.whl" + hash: + value: "758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/5f/d4/e834d929be54bfadb1f3e3b931c38e956aaa3b235a46a3c764c26c774902/fasteners-0.19.tar.gz" + hash: + value: "b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/harlowja/fasteners.git" + revision: "" + path: "" + - id: "PyPI::filelock:3.15.4" + purl: "pkg:pypi/filelock@3.15.4" + declared_licenses: + - "The Unlicense (Unlicense)" + declared_licenses_processed: + spdx_expression: "Unlicense" + mapped: + The Unlicense (Unlicense): "Unlicense" + description: "A platform independent file lock." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ae/f0/48285f0262fe47103a4a45972ed2f9b93e4c80b8fd609fa98da78b2a5706/filelock-3.15.4-py3-none-any.whl" + hash: + value: "6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/08/dd/49e06f09b6645156550fb9aee9cc1e59aba7efbc972d665a1bd6ae0435d4/filelock-3.15.4.tar.gz" + hash: + value: "2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/tox-dev/py-filelock.git" + revision: "" + path: "" + - id: "PyPI::fingerprints:1.2.3" + purl: "pkg:pypi/fingerprints@1.2.3" + authors: + - "Friedrich Lindenberg " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A library to generate entity fingerprints." + homepage_url: "http://github.com/alephdata/fingerprints" + binary_artifact: + url: "https://files.pythonhosted.org/packages/7d/2b/24a2675458df250e144174b0d18d70ee031eed5c108256200a68aaf087f9/fingerprints-1.2.3-py2.py3-none-any.whl" + hash: + value: "b8f83ad13dcdadce94903383db3b9b062b85a3a86f54f9e26d8faa97313f20bf" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/cb/17/292aab0190d8c80647ad0961c3fb9830016541b3d54fa4a67b5327f4e922/fingerprints-1.2.3.tar.gz" + hash: + value: "1719f808ec8dd6c7b32c79129be3cc77dc2d2258008cd0236654862a86a78b97" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/alephdata/fingerprints.git" + revision: "" + path: "" + - id: "PyPI::ftfy:6.2.3" + purl: "pkg:pypi/ftfy@6.2.3" + authors: + - "Robyn Speer " + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Fixes mojibake and other problems with Unicode, after the fact" + homepage_url: "https://ftfy.readthedocs.io/en/latest/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ed/46/14d230ad057048aea7ccd2f96a80905830866d281ea90a6662a825490659/ftfy-6.2.3-py3-none-any.whl" + hash: + value: "f15761b023f3061a66207d33f0c0149ad40a8319fd16da91796363e2c049fdf8" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/da/a9/59f4354257e8350a25be1774021991fb3a99a2fb87d0c1f367592548aed3/ftfy-6.2.3.tar.gz" + hash: + value: "79b505988f29d577a58a9069afe75553a02a46e42de6091c0660cdc67812badc" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::furo:2024.8.6" + purl: "pkg:pypi/furo@2024.8.6" + authors: + - "Pradyun Gedam " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A clean customisable Sphinx documentation theme." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/27/48/e791a7ed487dbb9729ef32bb5d1af16693d8925f4366befef54119b2e576/furo-2024.8.6-py3-none-any.whl" + hash: + value: "6cd97c58b47813d3619e63e9081169880fbe331f0ca883c871ff1f3f11814f5c" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/a0/e2/d351d69a9a9e4badb4a5be062c2d0e87bd9e6c23b5e57337fef14bef34c8/furo-2024.8.6.tar.gz" + hash: + value: "b63e4cee8abfc3136d3bc03a3d45a76a850bada4d6374d24c1716b0e01394a01" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::gemfileparser2:0.9.3" + purl: "pkg:pypi/gemfileparser2@0.9.3" + authors: + - "Balasankar C, Rohit Potter, nexB. Inc. and others " + declared_licenses: + - "GPL-3.0-or-later OR MIT" + declared_licenses_processed: + spdx_expression: "GPL-3.0-or-later OR MIT" + description: "Parse Ruby Gemfile, .gemspec and Cocoapod .podspec files using\ + \ Python." + homepage_url: "https://github.com/nexB/gemfileparser2" + binary_artifact: + url: "https://files.pythonhosted.org/packages/c4/77/1478ebca9228029ba6df583704bc0f4bb05b8def0608f683bb09690819f9/gemfileparser2-0.9.3-py3-none-any.whl" + hash: + value: "6d19bd99a81dff98dafed4437f5194a383b4b22d6be1de2c92cb134a5a598152" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d3/39/2ad53fa5b9148632825aa28d1c7f9e96a092066779a097347e6941efcfc0/gemfileparser2-0.9.3.tar.gz" + hash: + value: "04528964e7f45b66f460d6ca2309eb9a8286bed3fc03a47d3eb52dee4602fc39" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/gemfileparser2.git" + revision: "" + path: "" + - id: "PyPI::google-api-core:2.19.1" + purl: "pkg:pypi/google-api-core@2.19.1" + authors: + - "Google LLC " + declared_licenses: + - "Apache 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "Google API client core library" + homepage_url: "https://github.com/googleapis/python-api-core" + binary_artifact: + url: "https://files.pythonhosted.org/packages/44/99/daa3541e8ecd7d8b7907b714ba92126097a976b5b3dbabdb5febdcf08554/google_api_core-2.19.1-py3-none-any.whl" + hash: + value: "f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/c2/41/42a127bf163d9bf1f21540a3bf41c69b231b88707d8d753680b8878201a6/google-api-core-2.19.1.tar.gz" + hash: + value: "f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/googleapis/python-api-core.git" + revision: "" + path: "" + - id: "PyPI::google-auth:2.33.0" + purl: "pkg:pypi/google-auth@2.33.0" + authors: + - "Google Cloud Platform " + declared_licenses: + - "Apache 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "Google Authentication Library" + homepage_url: "https://github.com/googleapis/google-auth-library-python" + binary_artifact: + url: "https://files.pythonhosted.org/packages/60/57/0f37c6f35847e26b7bea7d5e4f069cf037fd792cf8b67206311761e7bb92/google_auth-2.33.0-py2.py3-none-any.whl" + hash: + value: "8eff47d0d4a34ab6265c50a106a3362de6a9975bb08998700e389f857e4d39df" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/28/4d/626b37c6bcc1f211aef23f47c49375072c0cb19148627d98c85e099acbc8/google_auth-2.33.0.tar.gz" + hash: + value: "d6a52342160d7290e334b4d47ba390767e4438ad0d45b7630774533e82655b95" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/googleapis/google-auth-library-python.git" + revision: "" + path: "" + - id: "PyPI::googleapis-common-protos:1.63.2" + purl: "pkg:pypi/googleapis-common-protos@1.63.2" + authors: + - "Google LLC " + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Common protobufs used in Google APIs" + homepage_url: "https://github.com/googleapis/python-api-common-protos" + binary_artifact: + url: "https://files.pythonhosted.org/packages/02/48/87422ff1bddcae677fb6f58c97f5cfc613304a5e8ce2c3662760199c0a84/googleapis_common_protos-1.63.2-py2.py3-none-any.whl" + hash: + value: "27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/0b/1a/41723ae380fa9c561cbe7b61c4eef9091d5fe95486465ccfc84845877331/googleapis-common-protos-1.63.2.tar.gz" + hash: + value: "27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/googleapis/python-api-common-protos.git" + revision: "" + path: "" + - id: "PyPI::grpcio:1.65.5" + purl: "pkg:pypi/grpcio@1.65.5" + authors: + - "The gRPC Authors " + declared_licenses: + - "Apache License 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "HTTP/2-based RPC framework" + homepage_url: "https://grpc.io" + binary_artifact: + url: "https://files.pythonhosted.org/packages/99/6a/d9021f91eacf30e6410f4d1809517a950f0e8b9ccd9f1a0afa05b0d1c07c/grpcio-1.65.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "c3655139d7be213c32c79ef6fb2367cae28e56ef68e39b1961c43214b457f257" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/6c/d8/1d8f1640649808db79b689d65b03556077d5504baad5ea64b167a5adedad/grpcio-1.65.5.tar.gz" + hash: + value: "ec6f219fb5d677a522b0deaf43cea6697b16f338cb68d009e30930c4aa0d2209" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/grpc/grpc.git" + revision: "" + path: "" + - id: "PyPI::grpcio-tools:1.65.5" + purl: "pkg:pypi/grpcio-tools@1.65.5" + authors: + - "The gRPC Authors " + declared_licenses: + - "Apache License 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "Protobuf code generator for gRPC" + homepage_url: "https://grpc.io" + binary_artifact: + url: "https://files.pythonhosted.org/packages/24/8f/e25a413bfbe9d8d4008f7bec870621263e0d014be3ebcdd07538d04fe214/grpcio_tools-1.65.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "777243e4f7152da9d226d9cc1e6d7c2b94335e267c618260e6255a063bb7dfcb" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/78/2b/5979958c17f0f54fab1b3707a060d2780bd711698d1dc524b2208bfd8102/grpcio_tools-1.65.5.tar.gz" + hash: + value: "7c3a47ad0070bc907c7818caf55aa1948e9282d24e27afd21015872a25594bc7" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/grpc/grpc.git" + revision: "master" + path: "tools/distrib/python/grpcio_tools" + - id: "PyPI::html5lib:1.1" + purl: "pkg:pypi/html5lib@1.1" + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "HTML parser based on the WHATWG HTML specification" + homepage_url: "https://github.com/html5lib/html5lib-python" + binary_artifact: + url: "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl" + hash: + value: "0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz" + hash: + value: "b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/html5lib/html5lib-python.git" + revision: "" + path: "" + - id: "PyPI::idna:3.7" + purl: "pkg:pypi/idna@3.7" + authors: + - "Kim Davies " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "Internationalized Domain Names in Applications (IDNA)" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl" + hash: + value: "82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/21/ed/f86a79a07470cb07819390452f178b3bef1d375f2ec021ecfc709fc7cf07/idna-3.7.tar.gz" + hash: + value: "028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/kjd/idna.git" + revision: "" + path: "" + - id: "PyPI::imagesize:1.4.1" + purl: "pkg:pypi/imagesize@1.4.1" + authors: + - "Yoshiki Shibukawa " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Getting image size from png/jpeg/jpeg2000/gif file" + homepage_url: "https://github.com/shibukawa/imagesize_py" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl" + hash: + value: "0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz" + hash: + value: "69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/shibukawa/imagesize_py.git" + revision: "" + path: "" + - id: "PyPI::importlib-metadata:8.2.0" + purl: "pkg:pypi/importlib-metadata@8.2.0" + authors: + - "\"Jason R. Coombs\" " + declared_licenses: + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Read metadata from Python packages" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/82/47/bb25ec04985d0693da478797c3d8c1092b140f3a53ccb984fbbd38affa5b/importlib_metadata-8.2.0-py3-none-any.whl" + hash: + value: "11901fa0c2f97919b288679932bb64febaeacf289d18ac84dd68cb2e74213369" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/f6/a1/db39a513aa99ab3442010a994eef1cb977a436aded53042e69bee6959f74/importlib_metadata-8.2.0.tar.gz" + hash: + value: "72e8d4399996132204f9a16dcc751af254a48f8d1b20b9ff0f98d4a8f901e73d" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/python/importlib_metadata.git" + revision: "" + path: "" + - id: "PyPI::importlib-resources:6.4.3" + purl: "pkg:pypi/importlib-resources@6.4.3" + authors: + - "Barry Warsaw " + declared_licenses: + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Read resources from Python packages" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/bc/8b/e848c888201b211159cfceaac65cc3bc1e32ed9ab6ca30366c43e5f1969b/importlib_resources-6.4.3-py3-none-any.whl" + hash: + value: "2d6dfe3b9e055f72495c2085890837fc8c758984e209115c8792bddcb762cd93" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/61/b3/0412c28d21e31447e97728efcf8913afe1936692917629e6bdb847563484/importlib_resources-6.4.3.tar.gz" + hash: + value: "4a202b9b9d38563b46da59221d77bb73862ab5d79d461307bcb826d725448b98" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/python/importlib_resources.git" + revision: "" + path: "" + - id: "PyPI::intbitset:3.1.0" + purl: "pkg:pypi/intbitset@3.1.0" + authors: + - "Invenio collaboration, maintained by Philippe Ombredanne and AboutCode.org\ + \ " + declared_licenses: + - "GNU Lesser General Public License v3 or later (LGPLv3+)" + - "LGPL-3.0-or-later" + declared_licenses_processed: + spdx_expression: "LGPL-3.0-or-later" + mapped: + GNU Lesser General Public License v3 or later (LGPLv3+): "LGPL-3.0-or-later" + description: "C-based extension implementing fast integer bit sets." + homepage_url: "http://github.com/inveniosoftware-contrib/intbitset/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/24/70/b1ac992230c58501d31c6d6b50f61cf2ce72eb3a6890fc5df3d7999c5b80/intbitset-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "353ea6b4c2f2c0aba4bd7b92bf6116fa82b16de9255b8b11a3d799c9cc0b640e" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/17/1e/a0de4407565ae27f5c6352392f72c0a1238f9f28b07f8cd34fbc716b0bf6/intbitset-3.1.0.tar.gz" + hash: + value: "6e83c5ba7fda2520aa8565428bbaf842deb7293d665f3cd8281cb39254d2ff71" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/inveniosoftware-contrib/intbitset.git" + revision: "" + path: "" + - id: "PyPI::isodate:0.6.1" + purl: "pkg:pypi/isodate@0.6.1" + authors: + - "Gerhard Weis " + declared_licenses: + - "BSD" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "An ISO 8601 date/time/duration parser and formatter" + homepage_url: "https://github.com/gweis/isodate/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/b6/85/7882d311924cbcfc70b1890780763e36ff0b140c7e51c110fc59a532f087/isodate-0.6.1-py2.py3-none-any.whl" + hash: + value: "0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/db/7a/c0a56c7d56c7fa723988f122fa1f1ccf8c5c4ccc48efad0d214b49e5b1af/isodate-0.6.1.tar.gz" + hash: + value: "48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/gweis/isodate.git" + revision: "" + path: "" + - id: "PyPI::jaraco-classes:3.4.0" + purl: "pkg:pypi/jaraco-classes@3.4.0" + authors: + - "Jason R. Coombs " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Utility functions for Python class constructs" + homepage_url: "https://github.com/jaraco/jaraco.classes" + binary_artifact: + url: "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl" + hash: + value: "f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz" + hash: + value: "47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jaraco/jaraco.classes.git" + revision: "" + path: "" + - id: "PyPI::jaraco-context:5.3.0" + purl: "pkg:pypi/jaraco-context@5.3.0" + authors: + - "Jason R. Coombs " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Useful decorators and context managers" + homepage_url: "https://github.com/jaraco/jaraco.context" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d2/40/11b7bc1898cf1dcb87ccbe09b39f5088634ac78bb25f3383ff541c2b40aa/jaraco.context-5.3.0-py3-none-any.whl" + hash: + value: "3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/c9/60/e83781b07f9a66d1d102a0459e5028f3a7816fdd0894cba90bee2bbbda14/jaraco.context-5.3.0.tar.gz" + hash: + value: "c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jaraco/jaraco.context.git" + revision: "" + path: "" + - id: "PyPI::jaraco-functools:4.0.2" + purl: "pkg:pypi/jaraco-functools@4.0.2" + authors: + - "\"Jason R. Coombs\" " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Functools like those found in stdlib" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/b1/54/7623e24ffc63730c3a619101361b08860c6b7c7cfc1aef6edb66d80ed708/jaraco.functools-4.0.2-py3-none-any.whl" + hash: + value: "c9d16a3ed4ccb5a889ad8e0b7a343401ee5b2a71cee6ed192d3f68bc351e94e3" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/03/b1/6ca3c2052e584e9908a2c146f00378939b3c51b839304ab8ef4de067f042/jaraco_functools-4.0.2.tar.gz" + hash: + value: "3460c74cd0d32bf82b9576bbb3527c4364d5b27a21f5158a62aed6c4b42e23f5" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jaraco/jaraco.functools.git" + revision: "" + path: "" + - id: "PyPI::javaproperties:0.8.1" + purl: "pkg:pypi/javaproperties@0.8.1" + authors: + - "John Thorvald Wodder II " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Read & write Java .properties files" + homepage_url: "https://github.com/jwodder/javaproperties" + binary_artifact: + url: "https://files.pythonhosted.org/packages/47/e8/c244dd03cecdebaf8116c93afaa1c72c8d4833f078a5d35e00c3d2c3be64/javaproperties-0.8.1-py3-none-any.whl" + hash: + value: "0e9b43334d6c1a9bffe34e2ece52588e21a7e099869bdaa481a5c6498774e18e" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/68/52/d7db7b671e2d4596c759fb526864837677c1562462e45f0ba46aef9a28c5/javaproperties-0.8.1.tar.gz" + hash: + value: "9dcba389effe67d3f906bbdcc64b8ef2ee8eac00072406784ea636bb6ba56061" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jwodder/javaproperties.git" + revision: "" + path: "" + - id: "PyPI::jeepney:0.8.0" + purl: "pkg:pypi/jeepney@0.8.0" + authors: + - "Thomas Kluyver " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Low-level, pure Python DBus protocol wrapper." + homepage_url: "https://gitlab.com/takluyver/jeepney" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ae/72/2a1e2290f1ab1e06f71f3d0f1646c9e4634e70e1d37491535e19266e8dc9/jeepney-0.8.0-py3-none-any.whl" + hash: + value: "c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d6/f4/154cf374c2daf2020e05c3c6a03c91348d59b23c5366e968feb198306fdf/jeepney-0.8.0.tar.gz" + hash: + value: "5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://gitlab.com/takluyver/jeepney.git" + revision: "" + path: "" + - id: "PyPI::jinja2:3.1.4" + purl: "pkg:pypi/jinja2@3.1.4" + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "A very fast and expressive template engine." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/31/80/3a54838c3fb461f6fec263ebf3a3a41771bd05190238de3486aae8540c36/jinja2-3.1.4-py3-none-any.whl" + hash: + value: "bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/ed/55/39036716d19cab0747a5020fc7e907f362fbf48c984b14e62127f7e68e5d/jinja2-3.1.4.tar.gz" + hash: + value: "4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pallets/jinja.git" + revision: "" + path: "" + - id: "PyPI::jsonstreams:0.6.0" + purl: "pkg:pypi/jsonstreams@0.6.0" + authors: + - "Dylan Baker " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A JSON streaming writer" + homepage_url: "https://github.com/dcbaker/jsonstreams" + binary_artifact: + url: "https://files.pythonhosted.org/packages/af/be/233b55906cc033b890c2e4593077bc10c7e09257c46f5253dd9b2850f3f4/jsonstreams-0.6.0-py2.py3-none-any.whl" + hash: + value: "b2e609c2bc17eec77fe26dae4d32556ba59dafbbff30c9a4909f2e19fa5bb000" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/85/8c/01333839805428590015bb4cbc3b730876609e536954eb1140d24b410bd0/jsonstreams-0.6.0.tar.gz" + hash: + value: "721cda7391e9415b7b15cebd6cf92fc7f8788ca211eda7d64162a066ee45a72e" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/dcbaker/jsonstreams.git" + revision: "" + path: "" + - id: "PyPI::keyring:25.3.0" + purl: "pkg:pypi/keyring@25.3.0" + authors: + - "Kang Zhang " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Store and access your passwords safely." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/63/42/ea8c9726e5ee5ff0731978aaf7cd5fa16674cf549c46279b279d7167c2b4/keyring-25.3.0-py3-none-any.whl" + hash: + value: "8d963da00ccdf06e356acd9bf3b743208878751032d8599c6cc89eb51310ffae" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/32/30/bfdde7294ba6bb2f519950687471dc6a0996d4f77ab30d75c841fa4994ed/keyring-25.3.0.tar.gz" + hash: + value: "8d85a1ea5d6db8515b59e1c5d1d1678b03cf7fc8b8dcfb1651e8c4a524eb42ef" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jaraco/keyring.git" + revision: "" + path: "" + - id: "PyPI::license-expression:30.3.1" + purl: "pkg:pypi/license-expression@30.3.1" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + description: "license-expression is a comprehensive utility library to parse,\ + \ compare, simplify and normalize license expressions (such as SPDX license\ + \ expressions) using boolean logic." + homepage_url: "https://github.com/aboutcode-org/license-expression" + binary_artifact: + url: "https://files.pythonhosted.org/packages/91/84/a7cf5dfa141501a20cb63595f02edfe38e0db2e3cc34e4f3cd273cc285df/license_expression-30.3.1-py3-none-any.whl" + hash: + value: "97904b9185c7bbb1e98799606fa7424191c375e70ba63a524b6f7100e42ddc46" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/57/8b/dbe230196eee2de208ba87dcfae69c46db9d7ed70e2f30f143bf994ee075/license_expression-30.3.1.tar.gz" + hash: + value: "60d5bec1f3364c256a92b9a08583d7ea933c7aa272c8d36d04144a89a3858c01" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/aboutcode-org/license-expression.git" + revision: "" + path: "" + - id: "PyPI::lxml:5.3.0" + purl: "pkg:pypi/lxml@5.3.0" + authors: + - "lxml dev team " + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "Powerful and Pythonic XML processing library combining libxml2/libxslt\ + \ with the ElementTree API." + homepage_url: "https://lxml.de/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/67/a4/1f5fbd3f58d4069000522196b0b776a014f3feec1796da03e495cf23532d/lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/e7/6b/20c3a4b24751377aaa6307eb230b66701024012c29dd374999cc92983269/lxml-5.3.0.tar.gz" + hash: + value: "4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/lxml/lxml.git" + revision: "" + path: "" + - id: "PyPI::markdown-it-py:3.0.0" + purl: "pkg:pypi/markdown-it-py@3.0.0" + authors: + - "Chris Sewell " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Python port of markdown-it. Markdown parsing, done right!" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl" + hash: + value: "355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz" + hash: + value: "e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::markupsafe:2.1.5" + purl: "pkg:pypi/markupsafe@2.1.5" + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "Safely add untrusted strings to HTML/XML markup." + homepage_url: "https://palletsprojects.com/p/markupsafe/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/97/18/c30da5e7a0e7f4603abfc6780574131221d9148f323752c2755d48abad30/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/87/5b/aae44c6655f3801e81aa3eef09dbbf012431987ba564d7231722f68df02d/MarkupSafe-2.1.5.tar.gz" + hash: + value: "d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pallets/markupsafe.git" + revision: "" + path: "" + - id: "PyPI::mdurl:0.1.2" + purl: "pkg:pypi/mdurl@0.1.2" + authors: + - "Taneli Hukkinen " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Markdown URL utilities" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl" + hash: + value: "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz" + hash: + value: "bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::more-itertools:10.4.0" + purl: "pkg:pypi/more-itertools@10.4.0" + authors: + - "Erik Rose " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "More routines for operating on iterables, beyond itertools" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d8/0b/6a51175e1395774449fca317fb8861379b7a2d59be411b8cce3d19d6ce78/more_itertools-10.4.0-py3-none-any.whl" + hash: + value: "0f7d9f83a0a8dcfa8a2694a770590d98a67ea943e3d9f5298309a484758c4e27" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/92/0d/ad6a82320cb8eba710fd0dceb0f678d5a1b58d67d03ae5be14874baa39e0/more-itertools-10.4.0.tar.gz" + hash: + value: "fe0e63c4ab068eac62410ab05cccca2dc71ec44ba8ef29916a0090df061cf923" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::mutf8:1.0.6" + purl: "pkg:pypi/mutf8@1.0.6" + authors: + - "Tyler Kennedy " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Fast MUTF-8 encoder & decoder" + homepage_url: "http://github.com/TkTech/mutf8" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://files.pythonhosted.org/packages/ca/31/3c57313757b3a47dcf32d2a9bad55d913b797efc8814db31bed8a7142396/mutf8-1.0.6.tar.gz" + hash: + value: "1bbbefb67c2e5a57104750bb04b0912200b57b2fa9841be245279e83859cb346" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/TkTech/mutf8.git" + revision: "" + path: "" + - id: "PyPI::nh3:0.2.18" + purl: "pkg:pypi/nh3@0.2.18" + authors: + - "messense >" + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Python bindings to the ammonia HTML sanitization library." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/1b/63/6ab90d0e5225ab9780f6c9fb52254fa36b52bb7c188df9201d05b647e5e1/nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/62/73/10df50b42ddb547a907deeb2f3c9823022580a7a47281e8eae8e003a9639/nh3-0.2.18.tar.gz" + hash: + value: "94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/messense/nh3.git" + revision: "" + path: "" + - id: "PyPI::normality:2.5.0" + purl: "pkg:pypi/normality@2.5.0" + authors: + - "Friedrich Lindenberg " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Micro-library to normalize text strings" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ae/29/cdd620678624e76de4034d1d69eb978cae4a96983dde963586f711261196/normality-2.5.0-py2.py3-none-any.whl" + hash: + value: "d9f48daf32e351e88b9e372787c1da437df9d0d818aec6e2834b02102378df62" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/e0/12/6452229afa2331de60fe93324dd9e2eb6034cb2e2faf6867419d9c51d356/normality-2.5.0.tar.gz" + hash: + value: "a55133e972b81c4a3bf8b6dc419f262f94a4fd6f636297046f74d35c93abe153" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::packageurl-python:0.15.6" + purl: "pkg:pypi/packageurl-python@0.15.6" + authors: + - "the purl authors" + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A purl aka. Package URL parser and builder" + homepage_url: "https://github.com/package-url/packageurl-python" + binary_artifact: + url: "https://files.pythonhosted.org/packages/4b/ca/b598e18eb0820a0116690a960d85625aae50dae8ba58195e254e35c2289a/packageurl_python-0.15.6-py3-none-any.whl" + hash: + value: "a40210652c89022772a6c8340d6066f7d5dc67132141e5284a4db7a27d0a8ab0" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/56/c5/c0f3ac14fd44f9b344069397fbe79aad1fd2c69220d145447c6c29cb541d/packageurl_python-0.15.6.tar.gz" + hash: + value: "cbc89afd15d5f4d05db4f1b61297e5b97a43f61f28799f6d282aff467ed2ee96" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/package-url/packageurl-python.git" + revision: "" + path: "" + - id: "PyPI::packaging:24.1" + purl: "pkg:pypi/packaging@24.1" + authors: + - "Donald Stufft " + declared_licenses: + - "Apache Software License" + - "BSD License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + unmapped: + - "BSD License" + description: "Core utilities for Python packages" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl" + hash: + value: "5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/51/65/50db4dda066951078f0a96cf12f4b9ada6e4b811516bf0262c0f4f7064d4/packaging-24.1.tar.gz" + hash: + value: "026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pypa/packaging.git" + revision: "" + path: "" + - id: "PyPI::packvers:21.5" + purl: "pkg:pypi/packvers@21.5" + authors: + - "Donald Stufft and individual contributors " + declared_licenses: + - "Apache Software License" + - "BSD License" + - "BSD-2-Clause or Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0 AND (Apache-2.0 OR BSD-2-Clause)" + mapped: + Apache Software License: "Apache-2.0" + BSD License: "BSD-2-Clause" + BSD-2-Clause or Apache-2.0: "BSD-2-Clause OR Apache-2.0" + description: "Core utilities for Python packages. Fork to support LegacyVersion" + homepage_url: "https://github.com/nexB/packvers" + binary_artifact: + url: "https://files.pythonhosted.org/packages/00/0c/a57d44f7f970ea31dfcda7ffbe8509d087c2386a28b791867a8868fc66da/packvers-21.5-py3-none-any.whl" + hash: + value: "a05e4a2b0f2eecb49d2568bfe180168a99165ab5167aa791f82266e33740ac87" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/c5/1b/dfb8654324222e996a18297b2b2b21b637cad0a23d75134cc1fd129dd7c0/packvers-21.5.tar.gz" + hash: + value: "2d2758fc09d2c325414354b8478d649f878b52c38598517fba51c8623526ca79" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/packvers.git" + revision: "" + path: "" + - id: "PyPI::parameter-expansion-patched:0.3.1" + purl: "pkg:pypi/parameter-expansion-patched@0.3.1" + authors: + - "Michael A. Smith and Philippe Ombredanne " + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Shell parameter expansion in Python. Patched by co-maintainer\ + \ for a PyPI release." + homepage_url: "https://github.com/nexB/parameter-expansion-patched" + binary_artifact: + url: "https://files.pythonhosted.org/packages/6c/9f/2eb2762808faed5218faba5559415b5bb62b39376cf9a38acc01f9786481/parameter_expansion_patched-0.3.1-py3-none-any.whl" + hash: + value: "832f04bed2a81e32d9d233cbe27448a7a22edf9a744086dbd01066c41ad0f535" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/7e/15/0c6fa115b269418a0d53d4564809afb74684d8afa417323b406be26de08b/parameter-expansion-patched-0.3.1.tar.gz" + hash: + value: "ff5dbc89fbde582f3336562d196b710771e92baa7b6d59356a14b085a0b6740b" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/parameter-expansion-patched.git" + revision: "" + path: "" + - id: "PyPI::pdfminer-six:20240706" + purl: "pkg:pypi/pdfminer-six@20240706" + authors: + - "Yusuke Shinyama + Philippe Guglielmetti " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "PDF parser and analyzer" + homepage_url: "https://github.com/pdfminer/pdfminer.six" + binary_artifact: + url: "https://files.pythonhosted.org/packages/67/7d/44d6b90e5a293d3a975cefdc4e12a932ebba814995b2a07e37e599dd27c6/pdfminer.six-20240706-py3-none-any.whl" + hash: + value: "f4f70e74174b4b3542fcb8406a210b6e2e27cd0f0b5fd04534a8cc0d8951e38c" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/e3/37/63cb918ffa21412dd5d54e32e190e69bfc340f3d6aa072ad740bec9386bb/pdfminer.six-20240706.tar.gz" + hash: + value: "c631a46d5da957a9ffe4460c5dce21e8431dabb615fee5f9f4400603a58d95a6" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pdfminer/pdfminer.six.git" + revision: "" + path: "" + - id: "PyPI::pefile:2023.2.7" + purl: "pkg:pypi/pefile@2023.2.7" + authors: + - "Ero Carrera " + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "Python PE parsing module" + homepage_url: "https://github.com/erocarrera/pefile" + binary_artifact: + url: "https://files.pythonhosted.org/packages/55/26/d0ad8b448476d0a1e8d3ea5622dc77b916db84c6aa3cb1e1c0965af948fc/pefile-2023.2.7-py3-none-any.whl" + hash: + value: "da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/78/c5/3b3c62223f72e2360737fd2a57c30e5b2adecd85e70276879609a7403334/pefile-2023.2.7.tar.gz" + hash: + value: "82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/erocarrera/pefile.git" + revision: "" + path: "" + - id: "PyPI::pip-requirements-parser:32.0.1" + purl: "pkg:pypi/pip-requirements-parser@32.0.1" + authors: + - "The pip authors, nexB. Inc. and others " + declared_licenses: + - "MIT" + declared_licenses_processed: + spdx_expression: "MIT" + description: "pip requirements parser - a mostly correct pip requirements parsing\ + \ library because it uses pip's own code." + homepage_url: "https://github.com/nexB/pip-requirements-parser" + binary_artifact: + url: "https://files.pythonhosted.org/packages/54/d0/d04f1d1e064ac901439699ee097f58688caadea42498ec9c4b4ad2ef84ab/pip_requirements_parser-32.0.1-py3-none-any.whl" + hash: + value: "4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/5e/2a/63b574101850e7f7b306ddbdb02cb294380d37948140eecd468fae392b54/pip-requirements-parser-32.0.1.tar.gz" + hash: + value: "b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/pip-requirements-parser.git" + revision: "" + path: "" + - id: "PyPI::pkginfo:1.10.0" + purl: "pkg:pypi/pkginfo@1.10.0" + authors: + - "Tres Seaver, Agendaless Consulting " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Query metadata from sdists / bdists / installed packages." + homepage_url: "https://code.launchpad.net/~tseaver/pkginfo/trunk" + binary_artifact: + url: "https://files.pythonhosted.org/packages/56/09/054aea9b7534a15ad38a363a2bd974c20646ab1582a387a95b8df1bfea1c/pkginfo-1.10.0-py3-none-any.whl" + hash: + value: "889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/2f/72/347ec5be4adc85c182ed2823d8d1c7b51e13b9a6b0c1aae59582eca652df/pkginfo-1.10.0.tar.gz" + hash: + value: "5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::pkginfo2:30.0.0" + purl: "pkg:pypi/pkginfo2@30.0.0" + authors: + - "Maintained by nexB, Inc. Authored by Tres Seaver, Agendaless Consulting " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Query metadatdata from sdists / bdists / installed packages. Safer\ + \ fork of pkginfo to avoid doing arbitrary imports and eval()" + homepage_url: "https://github.com/nexB/pkginfo2" + binary_artifact: + url: "https://files.pythonhosted.org/packages/49/01/4e506c68c9ea09c702b1eac87e6d2cda6d6633e6ed42ec1f43662e246769/pkginfo2-30.0.0-py3-none-any.whl" + hash: + value: "f1558f3ff71c99e8f362b6d079c15ef334dfce8ab2bc623a992341baeb1e7248" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/90/8d/09cc1c99a30ac14050fc4e04e549e024be83ff72a7f63e75023501baf977/pkginfo2-30.0.0.tar.gz" + hash: + value: "5e1afbeb156febb407a9b5c16b51c5b4737c529eeda2b1607e1e277cf260669c" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/pkginfo2.git" + revision: "" + path: "" + - id: "PyPI::pluggy:1.5.0" + purl: "pkg:pypi/pluggy@1.5.0" + authors: + - "Holger Krekel " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "plugin and hook calling mechanisms for python" + homepage_url: "https://github.com/pytest-dev/pluggy" + binary_artifact: + url: "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl" + hash: + value: "44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz" + hash: + value: "2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pytest-dev/pluggy.git" + revision: "" + path: "" + - id: "PyPI::plugincode:32.0.0" + purl: "pkg:pypi/plugincode@32.0.0" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + description: "plugincode is a library that provides plugin functionality for\ + \ ScanCode toolkit." + homepage_url: "https://github.com/nexB/plugincode" + binary_artifact: + url: "https://files.pythonhosted.org/packages/27/6f/d38bd65c3bcb3787d6cf944c8458e45cd38f8dea0ef7587ff2998e786595/plugincode-32.0.0-py3-none-any.whl" + hash: + value: "344bb9943fcf4d6d05669c3c61efd4093fffa6a290fba7c5c11db15f2b51305e" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/cc/0d/e934e7ae95363f9504e78b5f962efd62add3659aa9ad7c79b6f34faa81e6/plugincode-32.0.0.tar.gz" + hash: + value: "4132d93b1755271c6e226c9da2e2044ff62ebcb873b5e958d66a8ddde9f345fa" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/plugincode.git" + revision: "" + path: "" + - id: "PyPI::ply:3.11" + purl: "pkg:pypi/ply@3.11" + authors: + - "David Beazley " + declared_licenses: + - "BSD" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + description: "Python Lex & Yacc" + homepage_url: "http://www.dabeaz.com/ply/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl" + hash: + value: "096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz" + hash: + value: "00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::progress:1.6" + purl: "pkg:pypi/progress@1.6" + authors: + - "Georgios Verigakis " + declared_licenses: + - "ISC" + - "ISC License (ISCL)" + declared_licenses_processed: + spdx_expression: "ISC" + mapped: + ISC License (ISCL): "ISC" + description: "Easy to use progress bars" + homepage_url: "http://github.com/verigak/progress/" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://files.pythonhosted.org/packages/2a/68/d8412d1e0d70edf9791cbac5426dc859f4649afc22f2abbeb0d947cf70fd/progress-1.6.tar.gz" + hash: + value: "c9c86e98b5c03fa1fe11e3b67c1feda4788b8d0fe7336c2ff7d5644ccfba34cd" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/verigak/progress.git" + revision: "" + path: "" + - id: "PyPI::proto-plus:1.24.0" + purl: "pkg:pypi/proto-plus@1.24.0" + authors: + - "Google LLC " + declared_licenses: + - "Apache 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "Proto Plus for Python" + homepage_url: "https://github.com/googleapis/proto-plus-python.git" + binary_artifact: + url: "https://files.pythonhosted.org/packages/7c/6f/db31f0711c0402aa477257205ce7d29e86a75cb52cd19f7afb585f75cda0/proto_plus-1.24.0-py3-none-any.whl" + hash: + value: "402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/3e/fc/e9a65cd52c1330d8d23af6013651a0bc50b6d76bcbdf91fae7cd19c68f29/proto-plus-1.24.0.tar.gz" + hash: + value: "30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/googleapis/proto-plus-python.git" + revision: "" + path: "" + - id: "PyPI::protobuf:5.27.3" + purl: "pkg:pypi/protobuf@5.27.3" + authors: + - "protobuf@googlegroups.com " + declared_licenses: + - "3-Clause BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + "3-Clause BSD License": "BSD-3-Clause" + description: "" + homepage_url: "https://developers.google.com/protocol-buffers/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/e1/94/d77bd282d3d53155147166c2bbd156f540009b0d7be24330f76286668b90/protobuf-5.27.3-py3-none-any.whl" + hash: + value: "8572c6533e544ebf6899c360e91d6bcbbee2549251643d32c52cf8a5de295ba5" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/1b/61/0671db2ab2aee7c92d6c1b617c39b30a4cd973950118da56d77e7f397a9d/protobuf-5.27.3.tar.gz" + hash: + value: "82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::publicsuffix2:2.20191221" + purl: "pkg:pypi/publicsuffix2@2.20191221" + authors: + - "nexB Inc., Tomaz Solc, David Wilson and others. " + declared_licenses: + - "MIT License" + - "MIT and MPL-2.0" + - "Mozilla Public License 2.0 (MPL 2.0)" + declared_licenses_processed: + spdx_expression: "MIT AND MIT AND MPL-2.0 AND MPL-2.0" + mapped: + MIT License: "MIT" + MIT and MPL-2.0: "MIT AND MPL-2.0" + Mozilla Public License 2.0 (MPL 2.0): "MPL-2.0" + description: "Get a public suffix for a domain name using the Public Suffix\ + \ List. Forked from and using the same API as the publicsuffix package." + homepage_url: "https://github.com/nexb/python-publicsuffix2" + binary_artifact: + url: "https://files.pythonhosted.org/packages/9d/16/053c2945c5e3aebeefb4ccd5c5e7639e38bc30ad1bdc7ce86c6d01707726/publicsuffix2-2.20191221-py2.py3-none-any.whl" + hash: + value: "786b5e36205b88758bd3518725ec8cfe7a8173f5269354641f581c6b80a99893" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/5a/04/1759906c4c5b67b2903f546de234a824d4028ef24eb0b1122daa43376c20/publicsuffix2-2.20191221.tar.gz" + hash: + value: "00f8cc31aa8d0d5592a5ced19cccba7de428ebca985db26ac852d920ddd6fe7b" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexb/python-publicsuffix2.git" + revision: "" + path: "" + - id: "PyPI::pyahocorasick:2.1.0" + purl: "pkg:pypi/pyahocorasick@2.1.0" + authors: + - "Wojciech Muła " + declared_licenses: + - "BSD License" + - "BSD-3-Clause and Public-Domain" + declared_licenses_processed: + unmapped: + - "BSD License" + - "BSD-3-Clause and Public-Domain" + description: "pyahocorasick is a fast and memory efficient library for exact\ + \ or approximate multi-pattern string search. With the ``ahocorasick.Automaton``\ + \ class, you can find multiple key string occurrences at once in some input\ + \ text. You can use it as a plain dict-like Trie or convert a Trie to an\ + \ automaton for efficient Aho-Corasick search. And pickle to disk for easy\ + \ reuse of large automatons. Implemented in C and tested on Python 3.6+. Works\ + \ on Linux, macOS and Windows. BSD-3-Cause license." + homepage_url: "http://github.com/WojciechMula/pyahocorasick" + binary_artifact: + url: "https://files.pythonhosted.org/packages/31/32/17ab57fe5abcf09d2f1ceb502143447be00658761d167118441e19a2b2c6/pyahocorasick-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "a9f2728ac77bab807ba65c6ef41be30358ef0c9bb6960c9fe070d43f7024cb91" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/06/2e/075c667c27ecf2c3ed6bf3c62649625cf1e7de7fd349f63b49b794460b71/pyahocorasick-2.1.0.tar.gz" + hash: + value: "4df4845c1149e9fa4aa33f0f0aa35f5a42957a43a3d6e447c9b44e679e2672ea" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/WojciechMula/pyahocorasick.git" + revision: "" + path: "" + - id: "PyPI::pyasn1:0.6.0" + purl: "pkg:pypi/pyasn1@0.6.0" + authors: + - "Ilya Etingof " + declared_licenses: + - "BSD License" + - "BSD-2-Clause" + declared_licenses_processed: + spdx_expression: "BSD-2-Clause" + mapped: + BSD License: "BSD-2-Clause" + description: "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs\ + \ (X.208)" + homepage_url: "https://github.com/pyasn1/pyasn1" + binary_artifact: + url: "https://files.pythonhosted.org/packages/23/7e/5f50d07d5e70a2addbccd90ac2950f81d1edd0783630651d9268d7f1db49/pyasn1-0.6.0-py2.py3-none-any.whl" + hash: + value: "cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/4a/a3/d2157f333900747f20984553aca98008b6dc843eb62f3a36030140ccec0d/pyasn1-0.6.0.tar.gz" + hash: + value: "3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pyasn1/pyasn1.git" + revision: "" + path: "" + - id: "PyPI::pyasn1-modules:0.4.0" + purl: "pkg:pypi/pyasn1-modules@0.4.0" + authors: + - "Ilya Etingof " + declared_licenses: + - "BSD" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "A collection of ASN.1-based protocols modules" + homepage_url: "https://github.com/pyasn1/pyasn1-modules" + binary_artifact: + url: "https://files.pythonhosted.org/packages/13/68/8906226b15ef38e71dc926c321d2fe99de8048e9098b5dfd38343011c886/pyasn1_modules-0.4.0-py3-none-any.whl" + hash: + value: "be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/f7/00/e7bd1dec10667e3f2be602686537969a7ac92b0a7c5165be2e5875dc3971/pyasn1_modules-0.4.0.tar.gz" + hash: + value: "831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pyasn1/pyasn1-modules.git" + revision: "" + path: "" + - id: "PyPI::pycparser:2.22" + purl: "pkg:pypi/pycparser@2.22" + authors: + - "Eli Bendersky " + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "C parser in Python" + homepage_url: "https://github.com/eliben/pycparser" + binary_artifact: + url: "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl" + hash: + value: "c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz" + hash: + value: "491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/eliben/pycparser.git" + revision: "" + path: "" + - id: "PyPI::pygmars:0.8.1" + purl: "pkg:pypi/pygmars@0.8.1" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + description: "Craft simple regex-based small language lexers and parsers. Build\ + \ parsers from grammars and accept Pygments lexers as an input. Derived from\ + \ NLTK." + homepage_url: "https://github.com/aboutcode-org/pygmars" + binary_artifact: + url: "https://files.pythonhosted.org/packages/74/4e/74766da813bf2c1388277be438361dbb411210684e320788a93e02a94218/pygmars-0.8.1-py3-none-any.whl" + hash: + value: "eda9534289500c19cc8ae318839683a8c917834299a2ba4ee8777588b73d509b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/47/e2/53c68dd20a22057272acb5a71b87cf0c80fb112e2dc3bd8b25d08825d050/pygmars-0.8.1.tar.gz" + hash: + value: "07b324a7d8702ba3aec2b0243c5fec0ed7986e4c7e6926098637ee6ee894dc2d" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/aboutcode-org/pygmars.git" + revision: "" + path: "" + - id: "PyPI::pygments:2.18.0" + purl: "pkg:pypi/pygments@2.18.0" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD License" + - "BSD-2-Clause" + declared_licenses_processed: + spdx_expression: "BSD-2-Clause" + mapped: + BSD License: "BSD-2-Clause" + description: "Pygments" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl" + hash: + value: "b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/8e/62/8336eff65bcbc8e4cb5d05b55faf041285951b6e80f33e2bff2024788f31/pygments-2.18.0.tar.gz" + hash: + value: "786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pygments/pygments.git" + revision: "" + path: "" + - id: "PyPI::pymaven-patch:0.3.2" + purl: "pkg:pypi/pymaven-patch@0.3.2" + authors: + - "Walter Scheper " + declared_licenses: + - "Apache-2" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache-2: "Apache-2.0" + description: "Python access to maven. nexB advanced patch." + homepage_url: "https://github.com/nexB/pymaven" + binary_artifact: + url: "https://files.pythonhosted.org/packages/39/9a/9e597fcd70da0c2e34a9b3c60df62f23a6972295caa68f6d482b57db937b/pymaven_patch-0.3.2-py3-none-any.whl" + hash: + value: "29a67d508e5d7a55c4359435e009ab87217ceb604a48caeb7b5b7d26b3099f65" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/f2/5a/5de519f07057838d4768ed7f04ceea0629b373e06aa96bddfa7bb8d78654/pymaven-patch-0.3.2.tar.gz" + hash: + value: "0cf7c93e89f01f0408eb656eec58cb4a228c95e03b3d47cb73d31f899055cd50" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/pymaven.git" + revision: "" + path: "" + - id: "PyPI::pyopenssl:24.2.1" + purl: "pkg:pypi/pyopenssl@24.2.1" + authors: + - "The pyOpenSSL developers " + declared_licenses: + - "Apache License, Version 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "Python wrapper module around the OpenSSL library" + homepage_url: "https://pyopenssl.org/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d9/dd/e0aa7ebef5168c75b772eda64978c597a9129b46be17779054652a7999e4/pyOpenSSL-24.2.1-py3-none-any.whl" + hash: + value: "967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/5d/70/ff56a63248562e77c0c8ee4aefc3224258f1856977e0c1472672b62dadb8/pyopenssl-24.2.1.tar.gz" + hash: + value: "4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pyca/pyopenssl.git" + revision: "" + path: "" + - id: "PyPI::pypac:0.16.4" + purl: "pkg:pypi/pypac@0.16.4" + authors: + - "Carson Lam <46059+carsonyl@users.noreply.github.com>" + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Proxy auto-config and auto-discovery for Python." + homepage_url: "https://github.com/carsonyl/pypac" + binary_artifact: + url: "https://files.pythonhosted.org/packages/a0/af/9d71907e51ee270f19f33210cfbc5b7bebc8ae900beecac2685f4cd4c3b5/PyPAC-0.16.4-py2.py3-none-any.whl" + hash: + value: "dc2b775c7a2c9c77b1351681fec729788b08b7c76e6d2a041fe35cf60ca493c6" + algorithm: "SHA-256" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/carsonyl/pypac.git" + revision: "" + path: "" + - id: "PyPI::pyparsing:3.1.2" + purl: "pkg:pypi/pyparsing@3.1.2" + authors: + - "Paul McGuire " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "pyparsing module - Classes and methods to define and execute parsing\ + \ grammars" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/9d/ea/6d76df31432a0e6fdf81681a895f009a4bb47b3c39036db3e1b528191d52/pyparsing-3.1.2-py3-none-any.whl" + hash: + value: "f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/46/3a/31fd28064d016a2182584d579e033ec95b809d8e220e74c4af6f0f2e8842/pyparsing-3.1.2.tar.gz" + hash: + value: "a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::pyproject-hooks:1.1.0" + purl: "pkg:pypi/pyproject-hooks@1.1.0" + authors: + - "Thomas Kluyver " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Wrappers to call pyproject.toml-based build backend hooks." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl" + hash: + value: "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/c7/07/6f63dda440d4abb191b91dc383b472dae3dd9f37e4c1e4a5c3db150531c6/pyproject_hooks-1.1.0.tar.gz" + hash: + value: "4b37730834edbd6bd37f26ece6b44802fb1c1ee2ece0e54ddff8bfc06db86965" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pypa/pyproject-hooks.git" + revision: "" + path: "" + - id: "PyPI::pyyaml:6.0.2" + purl: "pkg:pypi/pyyaml@6.0.2" + authors: + - "Kirill Simonov " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "YAML parser and emitter for Python" + homepage_url: "https://pyyaml.org/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + hash: + value: "3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz" + hash: + value: "d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/yaml/pyyaml.git" + revision: "" + path: "" + - id: "PyPI::rdflib:7.0.0" + purl: "pkg:pypi/rdflib@7.0.0" + authors: + - "Daniel 'eikeon' Krech " + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "RDFLib is a Python library for working with RDF, a simple yet\ + \ powerful language for representing information." + homepage_url: "https://github.com/RDFLib/rdflib" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d4/b0/7b7d8b5b0d01f1a0b12cc2e5038a868ef3a15825731b8a0d776cf47566c0/rdflib-7.0.0-py3-none-any.whl" + hash: + value: "0438920912a642c866a513de6fe8a0001bd86ef975057d6962c79ce4771687cd" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/0d/a3/63740490a392921a611cfc05b5b17bffd4259b3c9589c7904a4033b3d291/rdflib-7.0.0.tar.gz" + hash: + value: "9995eb8569428059b8c1affd26b25eac510d64f5043d9ce8c84e0d0036e995ae" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/RDFLib/rdflib.git" + revision: "" + path: "" + - id: "PyPI::readme-renderer:44.0" + purl: "pkg:pypi/readme-renderer@44.0" + authors: + - "The Python Packaging Authority " + declared_licenses: + - "Apache License, Version 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache License, Version 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "readme_renderer is a library for rendering readme descriptions\ + \ for Warehouse" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/e1/67/921ec3024056483db83953ae8e48079ad62b92db7880013ca77632921dd0/readme_renderer-44.0-py3-none-any.whl" + hash: + value: "2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/5a/a9/104ec9234c8448c4379768221ea6df01260cd6c2ce13182d4eac531c8342/readme_renderer-44.0.tar.gz" + hash: + value: "8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::requests:2.32.3" + purl: "pkg:pypi/requests@2.32.3" + authors: + - "Kenneth Reitz " + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Python HTTP for Humans." + homepage_url: "https://requests.readthedocs.io" + binary_artifact: + url: "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl" + hash: + value: "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz" + hash: + value: "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/psf/requests.git" + revision: "" + path: "" + - id: "PyPI::requests-file:2.1.0" + purl: "pkg:pypi/requests-file@2.1.0" + authors: + - "David Shea " + declared_licenses: + - "Apache 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "File transport adapter for Requests" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d7/25/dd878a121fcfdf38f52850f11c512e13ec87c2ea72385933818e5b6c15ce/requests_file-2.1.0-py2.py3-none-any.whl" + hash: + value: "cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/72/97/bf44e6c6bd8ddbb99943baf7ba8b1a8485bcd2fe0e55e5708d7fee4ff1ae/requests_file-2.1.0.tar.gz" + hash: + value: "0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::requests-toolbelt:1.0.0" + purl: "pkg:pypi/requests-toolbelt@1.0.0" + authors: + - "Ian Cordasco, Cory Benfield " + declared_licenses: + - "Apache 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "A utility belt for advanced users of python-requests" + homepage_url: "https://toolbelt.readthedocs.io/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl" + hash: + value: "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz" + hash: + value: "7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/requests/toolbelt.git" + revision: "" + path: "" + - id: "PyPI::rfc3986:2.0.0" + purl: "pkg:pypi/rfc3986@2.0.0" + authors: + - "Ian Stapleton Cordasco " + declared_licenses: + - "Apache 2.0" + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache 2.0: "Apache-2.0" + Apache Software License: "Apache-2.0" + description: "Validating URI References per RFC 3986" + homepage_url: "http://rfc3986.readthedocs.io" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl" + hash: + value: "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz" + hash: + value: "97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::rich:13.7.1" + purl: "pkg:pypi/rich@13.7.1" + authors: + - "Will McGugan " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Render rich text, tables, progress bars, syntax highlighting,\ + \ markdown and more to the terminal" + homepage_url: "https://github.com/Textualize/rich" + binary_artifact: + url: "https://files.pythonhosted.org/packages/87/67/a37f6214d0e9fe57f6ae54b2956d550ca8365857f42a1ce0392bb21d9410/rich-13.7.1-py3-none-any.whl" + hash: + value: "4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/b3/01/c954e134dc440ab5f96952fe52b4fdc64225530320a910473c1fe270d9aa/rich-13.7.1.tar.gz" + hash: + value: "9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/Textualize/rich.git" + revision: "" + path: "" + - id: "PyPI::rsa:4.9" + purl: "pkg:pypi/rsa@4.9" + authors: + - "Sybren A. Stüvel " + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Pure-Python RSA implementation" + homepage_url: "https://stuvel.eu/rsa" + binary_artifact: + url: "https://files.pythonhosted.org/packages/49/97/fa78e3d2f65c02c8e1268b9aba606569fe97f6c8f7c2d74394553347c145/rsa-4.9-py3-none-any.whl" + hash: + value: "90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/aa/65/7d973b89c4d2351d7fb232c2e452547ddfa243e93131e7cfa766da627b52/rsa-4.9.tar.gz" + hash: + value: "e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::saneyaml:0.6.1" + purl: "pkg:pypi/saneyaml@0.6.1" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + description: "Read and write readable YAML safely preserving order and avoiding\ + \ bad surprises with unwanted infered type conversions. This library is a\ + \ PyYaml wrapper with sane behaviour to read and write readable YAML safely,\ + \ typically when used for configuration." + homepage_url: "https://github.com/aboutcode-org/saneyaml" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ea/c0/b41733920cef3d87ee7d1fd5a618c7bb5240ba80dd2f29c73ec3416b3e04/saneyaml-0.6.1-py3-none-any.whl" + hash: + value: "60553363ac55433cef2bc1d6c5a1c9f6e2787e5f40e8c6fad5983eb701592c5b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/18/bb/b3ab128fe13964fc8da25ecbac82f9ed9beb59b2e04bfbef433886f1acb0/saneyaml-0.6.1.tar.gz" + hash: + value: "19cfbd8bf94d730998162c790fe5cec9abb5300cc5890fe37dc6dbcaa8fb16bb" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/aboutcode-org/saneyaml.git" + revision: "" + path: "" + - id: "PyPI::scancode-toolkit-mini:32.2.1" + purl: "pkg:pypi/scancode-toolkit-mini@32.2.1" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" + declared_licenses_processed: + spdx_expression: "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-copyleft\ + \ AND LicenseRef-scancode-other-permissive" + description: "ScanCode is a tool to scan code for license, copyright, package\ + \ and their documented dependencies and other interesting facts. scancode-toolkit-mini\ + \ is a special build that does not come with pre-built binary dependencies\ + \ by default. These are instead installed separately or with the extra_requires\ + \ scancode-toolkit-mini[full]" + homepage_url: "https://github.com/nexB/scancode-toolkit" + binary_artifact: + url: "https://files.pythonhosted.org/packages/29/72/5f6cce8d3a9a503a658535b22dbcd41c8cd2f59a625da1ffd0e2b2a6f98f/scancode_toolkit_mini-32.2.1-cp311-none-any.whl" + hash: + value: "d6d791d75f797107fd19b97e48754376b5d7ab8d847a4b2e5a8538f9c578b7ad" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/52/21/2e8e21c0413212d4c5457b1d1811691dc344b38b62429e995cf5f8e3218d/scancode-toolkit-mini-32.2.1.tar.gz" + hash: + value: "2a93e90b0797696bb2247fe8230e155d3fb0124408426b885b35bbf52854a00e" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/scancode-toolkit.git" + revision: "" + path: "" + - id: "PyPI::secretstorage:3.3.3" + purl: "pkg:pypi/secretstorage@3.3.3" + authors: + - "Dmitry Shachnev " + declared_licenses: + - "BSD 3-Clause License" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD 3-Clause License: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "Python bindings to FreeDesktop.org Secret Service API" + homepage_url: "https://github.com/mitya57/secretstorage" + binary_artifact: + url: "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl" + hash: + value: "f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/53/a4/f48c9d79cb507ed1373477dbceaba7401fd8a23af63b837fa61f1dcd3691/SecretStorage-3.3.3.tar.gz" + hash: + value: "2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/mitya57/secretstorage.git" + revision: "" + path: "" + - id: "PyPI::semantic-version:2.10.0" + purl: "pkg:pypi/semantic-version@2.10.0" + authors: + - "Raphaël Barrois " + declared_licenses: + - "BSD" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "A library implementing the 'SemVer' scheme." + homepage_url: "https://github.com/rbarrois/python-semanticversion" + binary_artifact: + url: "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl" + hash: + value: "de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/7d/31/f2289ce78b9b473d582568c234e104d2a342fd658cc288a7553d83bb8595/semantic_version-2.10.0.tar.gz" + hash: + value: "bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/rbarrois/python-semanticversion.git" + revision: "" + path: "" + - id: "PyPI::setuptools:72.2.0" + purl: "pkg:pypi/setuptools@72.2.0" + authors: + - "Python Packaging Authority " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Easily download, build, install, upgrade, and uninstall Python\ + \ packages" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/6e/ec/06715d912351edc453e37f93f3fc80dcffd5ca0e70386c87529aca296f05/setuptools-72.2.0-py3-none-any.whl" + hash: + value: "f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/ce/ef/013ded5b0d259f3fa636bf35de186f0061c09fbe124020ce6b8db68c83af/setuptools-72.2.0.tar.gz" + hash: + value: "80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pypa/setuptools.git" + revision: "" + path: "" + - id: "PyPI::six:1.16.0" + purl: "pkg:pypi/six@1.16.0" + authors: + - "Benjamin Peterson " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Python 2 and 3 compatibility utilities" + homepage_url: "https://github.com/benjaminp/six" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" + hash: + value: "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz" + hash: + value: "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/benjaminp/six.git" + revision: "" + path: "" + - id: "PyPI::snowballstemmer:2.2.0" + purl: "pkg:pypi/snowballstemmer@2.2.0" + authors: + - "Snowball Developers " + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "This package provides 29 stemmers for 28 languages generated from\ + \ Snowball algorithms." + homepage_url: "https://github.com/snowballstem/snowball" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl" + hash: + value: "c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/44/7b/af302bebf22c749c56c9c3e8ae13190b5b5db37a33d9068652e8f73b7089/snowballstemmer-2.2.0.tar.gz" + hash: + value: "09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/snowballstem/snowball.git" + revision: "" + path: "" + - id: "PyPI::soupsieve:2.6" + purl: "pkg:pypi/soupsieve@2.6" + authors: + - "Isaac Muse " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A modern CSS selector implementation for Beautiful Soup." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/d1/c2/fe97d779f3ef3b15f05c94a2f1e3d21732574ed441687474db9d342a7315/soupsieve-2.6-py3-none-any.whl" + hash: + value: "e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d7/ce/fbaeed4f9fb8b2daa961f90591662df6a86c1abf25c548329a86920aedfb/soupsieve-2.6.tar.gz" + hash: + value: "e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::spdx-tools:0.8.2" + purl: "pkg:pypi/spdx-tools@0.8.2" + authors: + - "\"Ahmed H. Ismail\" " + declared_licenses: + - "Apache Software License" + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "SPDX parser and tools." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/a2/a3/5d311af9214f65b0e7106f13d14b677563533f81b4953578023051f4f916/spdx_tools-0.8.2-py3-none-any.whl" + hash: + value: "8c336c873f9caaf110693a1d38c007031e67bea53aa4b881007b680be66de934" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/32/d8/a67445be5981469fdbaf7f765f53c920f699e7e512cc931b650a935c3199/spdx-tools-0.8.2.tar.gz" + hash: + value: "aea4ac9c2c375e7f439b1cef5ff32ef34914c083de0f61e08ed67cd3d9deb2a9" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::sphinx:8.0.2" + purl: "pkg:pypi/sphinx@8.0.2" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "Python documentation generator" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/4d/61/2ad169c6ff1226b46e50da0e44671592dbc6d840a52034a0193a99b28579/sphinx-8.0.2-py3-none-any.whl" + hash: + value: "56173572ae6c1b9a38911786e206a110c9749116745873feae4f9ce88e59391d" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/25/a7/3cc3d6dcad70aba2e32a3ae8de5a90026a0a2fdaaa0756925e3a120249b6/sphinx-8.0.2.tar.gz" + hash: + value: "0cce1ddcc4fd3532cf1dd283bc7d886758362c5c1de6598696579ce96d8ffa5b" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/sphinx-doc/sphinx.git" + revision: "" + path: "" + - id: "PyPI::sphinx-basic-ng:1.0.0b2" + purl: "pkg:pypi/sphinx-basic-ng@1.0.0b2" + authors: + - "Pradyun Gedam " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A modern skeleton for Sphinx themes." + homepage_url: "https://github.com/pradyunsg/sphinx-basic-ng" + binary_artifact: + url: "https://files.pythonhosted.org/packages/3c/dd/018ce05c532a22007ac58d4f45232514cd9d6dd0ee1dc374e309db830983/sphinx_basic_ng-1.0.0b2-py3-none-any.whl" + hash: + value: "eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz" + hash: + value: "9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pradyunsg/sphinx-basic-ng.git" + revision: "" + path: "" + - id: "PyPI::sphinxcontrib-applehelp:2.0.0" + purl: "pkg:pypi/sphinxcontrib-applehelp@2.0.0" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple\ + \ help books" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl" + hash: + value: "4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz" + hash: + value: "2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/sphinx-doc/sphinxcontrib-applehelp.git" + revision: "" + path: "" + - id: "PyPI::sphinxcontrib-devhelp:2.0.0" + purl: "pkg:pypi/sphinxcontrib-devhelp@2.0.0" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp\ + \ documents" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl" + hash: + value: "aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz" + hash: + value: "411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/sphinx-doc/sphinxcontrib-devhelp.git" + revision: "" + path: "" + - id: "PyPI::sphinxcontrib-htmlhelp:2.1.0" + purl: "pkg:pypi/sphinxcontrib-htmlhelp@2.1.0" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "======================" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl" + hash: + value: "166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz" + hash: + value: "c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/sphinx-doc/sphinxcontrib-htmlhelp.git" + revision: "" + path: "" + - id: "PyPI::sphinxcontrib-jsmath:1.0.1" + purl: "pkg:pypi/sphinxcontrib-jsmath@1.0.1" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "A sphinx extension which renders display math in HTML via JavaScript" + homepage_url: "http://sphinx-doc.org/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl" + hash: + value: "2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz" + hash: + value: "a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::sphinxcontrib-qthelp:2.0.0" + purl: "pkg:pypi/sphinxcontrib-qthelp@2.0.0" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp\ + \ documents" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl" + hash: + value: "b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz" + hash: + value: "4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/sphinx-doc/sphinxcontrib-qthelp.git" + revision: "" + path: "" + - id: "PyPI::sphinxcontrib-serializinghtml:2.0.0" + purl: "pkg:pypi/sphinxcontrib-serializinghtml@2.0.0" + authors: + - "Georg Brandl " + declared_licenses: + - "BSD License" + declared_licenses_processed: + unmapped: + - "BSD License" + description: "sphinxcontrib-serializinghtml is a sphinx extension which outputs\ + \ \"serialized\" HTML files (json and pickle)" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl" + hash: + value: "6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz" + hash: + value: "e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/sphinx-doc/sphinxcontrib-serializinghtml.git" + revision: "" + path: "" + - id: "PyPI::text-unidecode:1.3" + purl: "pkg:pypi/text-unidecode@1.3" + authors: + - "Mikhail Korobov " + declared_licenses: + - "Artistic License" + - "GNU General Public License (GPL)" + - "GNU General Public License v2 or later (GPLv2+)" + declared_licenses_processed: + spdx_expression: "Artistic-2.0 AND GPL-2.0-or-later AND GPL-3.0-or-later" + mapped: + Artistic License: "Artistic-2.0" + GNU General Public License (GPL): "GPL-3.0-or-later" + GNU General Public License v2 or later (GPLv2+): "GPL-2.0-or-later" + description: "The most basic Text::Unidecode port" + homepage_url: "https://github.com/kmike/text-unidecode/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl" + hash: + value: "1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz" + hash: + value: "bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/kmike/text-unidecode.git" + revision: "" + path: "" + - id: "PyPI::tldextract:5.1.2" + purl: "pkg:pypi/tldextract@5.1.2" + authors: + - "John Kurkowski " + declared_licenses: + - "BSD License" + - "BSD-3-Clause" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD License: "BSD-3-Clause" + description: "Accurately separates a URL's subdomain, domain, and public suffix,\ + \ using the Public Suffix List (PSL). By default, this includes the public\ + \ ICANN TLDs and their exceptions. You can optionally support the Public Suffix\ + \ List's private domains as well." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/fc/6d/8eaafb735b39c4ab3bb8fe4324ef8f0f0af27a7df9bb4cd503927bd5475d/tldextract-5.1.2-py3-none-any.whl" + hash: + value: "4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/db/ed/c92a5d6edaafec52f388c2d2946b4664294299cebf52bb1ef9cbc44ae739/tldextract-5.1.2.tar.gz" + hash: + value: "c9e17f756f05afb5abac04fe8f766e7e70f9fe387adb1859f0f52408ee060200" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + - id: "PyPI::toml:0.10.2" + purl: "pkg:pypi/toml@0.10.2" + authors: + - "William Pearson " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Python Library for Tom's Obvious, Minimal Language" + homepage_url: "https://github.com/uiri/toml" + binary_artifact: + url: "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl" + hash: + value: "806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz" + hash: + value: "b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/uiri/toml.git" + revision: "" + path: "" + - id: "PyPI::twine:5.1.1" + purl: "pkg:pypi/twine@5.1.1" + authors: + - "Donald Stufft and individual contributors " + declared_licenses: + - "Apache Software License" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + mapped: + Apache Software License: "Apache-2.0" + description: "Collection of utilities for publishing packages on PyPI" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/5d/ec/00f9d5fd040ae29867355e559a94e9a8429225a0284a3f5f091a3878bfc0/twine-5.1.1-py3-none-any.whl" + hash: + value: "215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/77/68/bd982e5e949ef8334e6f7dcf76ae40922a8750aa2e347291ae1477a4782b/twine-5.1.1.tar.gz" + hash: + value: "9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pypa/twine.git" + revision: "" + path: "" + - id: "PyPI::typecode:30.0.2" + purl: "pkg:pypi/typecode@30.0.2" + authors: + - "nexB. Inc. and others " + declared_licenses: + - "Apache-2.0" + declared_licenses_processed: + spdx_expression: "Apache-2.0" + description: "Comprehensive filetype and mimetype detection using libmagic and\ + \ Pygments." + homepage_url: "https://github.com/nexB/typecode" + binary_artifact: + url: "https://files.pythonhosted.org/packages/a1/3c/6d135ccbbf42230d3bef43bc052fd6993171a71ce1ec868c2581d8f930ab/typecode-30.0.2-py3-none-any.whl" + hash: + value: "06b1bffa93525acf4b6a52393fd0ee20916f62ddd40c0f2d27ca75a08231a46c" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d6/ca/c93584110e501e579e8901c9ad0c5db60dc69e84dfaf2224d5f95fca067b/typecode-30.0.2.tar.gz" + hash: + value: "17689d20af0ae6116e797ef2c5de65f0ce809128cf0e68479b34bd6ba4bc3898" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/typecode.git" + revision: "" + path: "" + - id: "PyPI::typecode-libmagic:5.39.210531" + purl: "pkg:pypi/typecode-libmagic@5.39.210531" + authors: + - "nexB " + declared_licenses: + - "apache-2.0 AND bsd-simplified-darwin AND (bsd-simplified AND public-domain\ + \ AND bsd-new AND isc AND (bsd-new OR gpl-1.0-plus) AND bsd-original)" + declared_licenses_processed: + unmapped: + - "apache-2.0 AND bsd-simplified-darwin AND (bsd-simplified AND public-domain\ + \ AND bsd-new AND isc AND (bsd-new OR gpl-1.0-plus) AND bsd-original)" + description: "A ScanCode path provider plugin to provide a prebuilt native libmagic\ + \ binary and database." + homepage_url: "https://github.com/nexB/scancode-plugins" + binary_artifact: + url: "https://files.pythonhosted.org/packages/89/bc/135d2c5a345f1c52431dbc92c181003ba61bcd35e304d36387e33070a9c5/typecode_libmagic-5.39.210531-py3-none-manylinux1_x86_64.whl" + hash: + value: "ee001c8093dfa89a9d1fe6d9139ef9f367a1cb9af6fd02ffebf9246af994fbf7" + algorithm: "SHA-256" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/scancode-plugins.git" + revision: "" + path: "" + - id: "PyPI::uritools:4.0.3" + purl: "pkg:pypi/uritools@4.0.3" + authors: + - "Thomas Kemmer " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "URI parsing, classification and composition" + homepage_url: "https://github.com/tkem/uritools/" + binary_artifact: + url: "https://files.pythonhosted.org/packages/e6/17/5a4510d9ca9cc8be217ce359eb54e693dca81cf4d442308b282d5131b17d/uritools-4.0.3-py3-none-any.whl" + hash: + value: "bae297d090e69a0451130ffba6f2f1c9477244aa0a5543d66aed2d9f77d0dd9c" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/d3/43/4182fb2a03145e6d38698e38b49114ce59bc8c79063452eb585a58f8ce78/uritools-4.0.3.tar.gz" + hash: + value: "ee06a182a9c849464ce9d5fa917539aacc8edd2a4924d1b7aabeeecabcae3bc2" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/tkem/uritools.git" + revision: "" + path: "" + - id: "PyPI::urllib3:2.2.2" + purl: "pkg:pypi/urllib3@2.2.2" + authors: + - "Andrey Petrov " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "HTTP library with thread-safe connection pooling, file post, and\ + \ more." + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/ca/1c/89ffc63a9605b583d5df2be791a27bc1a42b7c32bab68d3c8f2f73a98cd4/urllib3-2.2.2-py3-none-any.whl" + hash: + value: "a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/43/6d/fa469ae21497ddc8bc93e5877702dca7cb8f911e337aca7452b5724f1bb6/urllib3-2.2.2.tar.gz" + hash: + value: "dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/urllib3/urllib3.git" + revision: "" + path: "" + - id: "PyPI::urlpy:0.5" + purl: "pkg:pypi/urlpy@0.5" + authors: + - "nexB Inc (based on code from Dan Lecocq)" + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Simple URL parsing, canonicalization and equivalence." + homepage_url: "http://github.com/nexB/urlpy" + binary_artifact: + url: "https://files.pythonhosted.org/packages/23/f0/43a8013e888f435c619f82b485ef8cf9fddfcceea7806d824b28d5ef8f76/urlpy-0.5-py2.py3-none-any.whl" + hash: + value: "841673d97e0dd7a4d7ba47abd49fa8e3a61709e189e40de1b04b150ce7c5ed9f" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/23/21/d176a137e4cc1ec4809534d116e9a06fc5fc0519077530ab5d040e356454/urlpy-0.5.tar.gz" + hash: + value: "e98ead47f4e422ca35080fd60a039f4546b7788bbba1b0a542a34c193dfba4bc" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/nexB/urlpy.git" + revision: "" + path: "" + - id: "PyPI::wcwidth:0.2.13" + purl: "pkg:pypi/wcwidth@0.2.13" + authors: + - "Jeff Quast " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Measures the displayed width of unicode strings in a terminal" + homepage_url: "https://github.com/jquast/wcwidth" + binary_artifact: + url: "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl" + hash: + value: "3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz" + hash: + value: "72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jquast/wcwidth.git" + revision: "" + path: "" + - id: "PyPI::webencodings:0.5.1" + purl: "pkg:pypi/webencodings@0.5.1" + authors: + - "Geoffrey Sneddon " + declared_licenses: + - "BSD" + - "BSD License" + declared_licenses_processed: + spdx_expression: "BSD-3-Clause" + mapped: + BSD: "BSD-3-Clause" + BSD License: "BSD-3-Clause" + description: "Character encoding aliases for legacy web content" + homepage_url: "https://github.com/SimonSapin/python-webencodings" + binary_artifact: + url: "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl" + hash: + value: "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz" + hash: + value: "b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/SimonSapin/python-webencodings.git" + revision: "" + path: "" + - id: "PyPI::wheel:0.44.0" + purl: "pkg:pypi/wheel@0.44.0" + authors: + - "Daniel Holth " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "A built-package format for Python" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/1b/d1/9babe2ccaecff775992753d8686970b1e2755d21c8a63be73aba7a4e7d77/wheel-0.44.0-py3-none-any.whl" + hash: + value: "2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/b7/a0/95e9e962c5fd9da11c1e28aa4c0d8210ab277b1ada951d2aee336b505813/wheel-0.44.0.tar.gz" + hash: + value: "a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/pypa/wheel.git" + revision: "" + path: "" + - id: "PyPI::xmltodict:0.13.0" + purl: "pkg:pypi/xmltodict@0.13.0" + authors: + - "Martin Blech " + declared_licenses: + - "MIT" + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Makes working with XML feel like you are working with JSON" + homepage_url: "https://github.com/martinblech/xmltodict" + binary_artifact: + url: "https://files.pythonhosted.org/packages/94/db/fd0326e331726f07ff7f40675cd86aa804bfd2e5016c727fa761c934990e/xmltodict-0.13.0-py2.py3-none-any.whl" + hash: + value: "aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/39/0d/40df5be1e684bbaecdb9d1e0e40d5d482465de6b00cbb92b84ee5d243c7f/xmltodict-0.13.0.tar.gz" + hash: + value: "341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/martinblech/xmltodict.git" + revision: "" + path: "" + - id: "PyPI::zipp:3.20.0" + purl: "pkg:pypi/zipp@3.20.0" + authors: + - "\"Jason R. Coombs\" " + declared_licenses: + - "MIT License" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + MIT License: "MIT" + description: "Backport of pathlib-compatible object wrapper for zip files" + homepage_url: "" + binary_artifact: + url: "https://files.pythonhosted.org/packages/da/cc/b9958af9f9c86b51f846d8487440af495ecf19b16e426fce1ed0b0796175/zipp-3.20.0-py3-none-any.whl" + hash: + value: "58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d" + algorithm: "SHA-256" + source_artifact: + url: "https://files.pythonhosted.org/packages/0e/af/9f2de5bd32549a1b705af7a7c054af3878816a1267cb389c03cc4f342a51/zipp-3.20.0.tar.gz" + hash: + value: "0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31" + algorithm: "SHA-256" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/jaraco/zipp.git" + revision: "" + path: "" + issues: + 'NPM::tests/data/package.json:': + - timestamp: "2024-08-17T19:07:21.849028339Z" + source: "NPM" + message: "NPM failed to resolve dependencies for path 'tests/data/package.json':\ + \ IllegalArgumentException: No lockfile found in 'tests/data'. This potentially\ + \ results in unstable versions of dependencies. To support this, enable\ + \ the 'allowDynamicVersions' option in 'config.yml'." + severity: "ERROR" + dependency_graphs: + NPM: + nodes: [] + edges: [] + PIP: + packages: + - "PyPI::alabaster:1.0.0" + - "PyPI::attrs:24.2.0" + - "PyPI::babel:2.16.0" + - "PyPI::backports-tarfile:1.2.0" + - "PyPI::banal:1.0.6" + - "PyPI::beartype:0.18.5" + - "PyPI::beautifulsoup4:4.12.3" + - "PyPI::binaryornot:0.4.4" + - "PyPI::boolean-py:4.0" + - "PyPI::build:1.2.1" + - "PyPI::cachetools:5.4.0" + - "PyPI::certifi:2024.7.4" + - "PyPI::cffi:1.17.0" + - "PyPI::chardet:5.2.0" + - "PyPI::charset-normalizer:3.3.2" + - "PyPI::click:8.1.7" + - "PyPI::colorama:0.4.6" + - "PyPI::commoncode:31.2.1" + - "PyPI::container-inspector:33.0.0" + - "PyPI::crc32c:2.6" + - "PyPI::cryptography:43.0.0" + - "PyPI::debian-inspector:31.1.0" + - "PyPI::dockerfile-parse:2.0.1" + - "PyPI::docutils:0.21.2" + - "PyPI::dparse2:0.7.0" + - "PyPI::dukpy:0.4.0" + - "PyPI::fasteners:0.19" + - "PyPI::filelock:3.15.4" + - "PyPI::fingerprints:1.2.3" + - "PyPI::ftfy:6.2.3" + - "PyPI::furo:2024.8.6" + - "PyPI::gemfileparser2:0.9.3" + - "PyPI::google-api-core:2.19.1" + - "PyPI::google-auth:2.33.0" + - "PyPI::googleapis-common-protos:1.63.2" + - "PyPI::grpcio-tools:1.65.5" + - "PyPI::grpcio:1.65.5" + - "PyPI::html5lib:1.1" + - "PyPI::idna:3.7" + - "PyPI::imagesize:1.4.1" + - "PyPI::importlib-metadata:8.2.0" + - "PyPI::importlib-resources:6.4.3" + - "PyPI::intbitset:3.1.0" + - "PyPI::isodate:0.6.1" + - "PyPI::jaraco-classes:3.4.0" + - "PyPI::jaraco-context:5.3.0" + - "PyPI::jaraco-functools:4.0.2" + - "PyPI::javaproperties:0.8.1" + - "PyPI::jeepney:0.8.0" + - "PyPI::jinja2:3.1.4" + - "PyPI::jsonstreams:0.6.0" + - "PyPI::keyring:25.3.0" + - "PyPI::license-expression:30.3.1" + - "PyPI::lxml:5.3.0" + - "PyPI::markdown-it-py:3.0.0" + - "PyPI::markupsafe:2.1.5" + - "PyPI::mdurl:0.1.2" + - "PyPI::more-itertools:10.4.0" + - "PyPI::mutf8:1.0.6" + - "PyPI::nh3:0.2.18" + - "PyPI::normality:2.5.0" + - "PyPI::packageurl-python:0.15.6" + - "PyPI::packaging:24.1" + - "PyPI::packvers:21.5" + - "PyPI::parameter-expansion-patched:0.3.1" + - "PyPI::pdfminer-six:20240706" + - "PyPI::pefile:2023.2.7" + - "PyPI::pip-requirements-parser:32.0.1" + - "PyPI::pkginfo2:30.0.0" + - "PyPI::pkginfo:1.10.0" + - "PyPI::pluggy:1.5.0" + - "PyPI::plugincode:32.0.0" + - "PyPI::ply:3.11" + - "PyPI::progress:1.6" + - "PyPI::proto-plus:1.24.0" + - "PyPI::protobuf:5.27.3" + - "PyPI::publicsuffix2:2.20191221" + - "PyPI::pyahocorasick:2.1.0" + - "PyPI::pyasn1-modules:0.4.0" + - "PyPI::pyasn1:0.6.0" + - "PyPI::pycparser:2.22" + - "PyPI::pygmars:0.8.1" + - "PyPI::pygments:2.18.0" + - "PyPI::pymaven-patch:0.3.2" + - "PyPI::pyopenssl:24.2.1" + - "PyPI::pypac:0.16.4" + - "PyPI::pyparsing:3.1.2" + - "PyPI::pyproject-hooks:1.1.0" + - "PyPI::pyyaml:6.0.2" + - "PyPI::rdflib:7.0.0" + - "PyPI::readme-renderer:44.0" + - "PyPI::requests-file:2.1.0" + - "PyPI::requests-toolbelt:1.0.0" + - "PyPI::requests:2.32.3" + - "PyPI::rfc3986:2.0.0" + - "PyPI::rich:13.7.1" + - "PyPI::rsa:4.9" + - "PyPI::saneyaml:0.6.1" + - "PyPI::scancode-toolkit-mini:32.2.1" + - "PyPI::secretstorage:3.3.3" + - "PyPI::semantic-version:2.10.0" + - "PyPI::setuptools:72.2.0" + - "PyPI::six:1.16.0" + - "PyPI::snowballstemmer:2.2.0" + - "PyPI::soupsieve:2.6" + - "PyPI::spdx-tools:0.8.2" + - "PyPI::sphinx-basic-ng:1.0.0b2" + - "PyPI::sphinx:8.0.2" + - "PyPI::sphinxcontrib-applehelp:2.0.0" + - "PyPI::sphinxcontrib-devhelp:2.0.0" + - "PyPI::sphinxcontrib-htmlhelp:2.1.0" + - "PyPI::sphinxcontrib-jsmath:1.0.1" + - "PyPI::sphinxcontrib-qthelp:2.0.0" + - "PyPI::sphinxcontrib-serializinghtml:2.0.0" + - "PyPI::text-unidecode:1.3" + - "PyPI::tldextract:5.1.2" + - "PyPI::toml:0.10.2" + - "PyPI::twine:5.1.1" + - "PyPI::typecode-libmagic:5.39.210531" + - "PyPI::typecode:30.0.2" + - "PyPI::uritools:4.0.3" + - "PyPI::urllib3:2.2.2" + - "PyPI::urlpy:0.5" + - "PyPI::wcwidth:0.2.13" + - "PyPI::webencodings:0.5.1" + - "PyPI::wheel:0.44.0" + - "PyPI::xmltodict:0.13.0" + - "PyPI::zipp:3.20.0" + scopes: + :data:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: + - root: 7 + - root: 19 + - root: 36 + - root: 73 + - root: 75 + - root: 93 + :docs-docs:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: + - root: 30 + :requirements-dev.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: + - root: 9 + - root: 35 + - root: 117 + - root: 125 + :requirements-scancode.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: + - root: 98 + - root: 118 + :requirements.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: + - root: 7 + - root: 19 + - root: 32 + - root: 36 + - root: 41 + - root: 73 + - root: 84 + - root: 85 + nodes: + - pkg: 13 + - pkg: 7 + - pkg: 19 + - pkg: 10 + - pkg: 79 + - pkg: 78 + - pkg: 96 + - pkg: 33 + - pkg: 75 + - pkg: 34 + - pkg: 74 + - pkg: 11 + - pkg: 14 + - pkg: 38 + - pkg: 121 + - pkg: 93 + - pkg: 32 + - pkg: 36 + - pkg: 41 + - pkg: 73 + - pkg: 80 + - pkg: 12 + - pkg: 20 + - pkg: 84 + - pkg: 58 + - pkg: 25 + - pkg: 27 + - pkg: 91 + - pkg: 115 + - pkg: 85 + - pkg: 1 + - pkg: 104 + - pkg: 6 + - pkg: 8 + - pkg: 15 + - pkg: 16 + - pkg: 88 + - pkg: 97 + - pkg: 114 + - pkg: 17 + - pkg: 22 + - pkg: 18 + - pkg: 21 + - pkg: 86 + - pkg: 63 + - pkg: 116 + - pkg: 24 + - pkg: 26 + - pkg: 4 + - pkg: 60 + - pkg: 28 + - pkg: 123 + - pkg: 29 + - pkg: 31 + - pkg: 102 + - pkg: 124 + - pkg: 37 + - pkg: 127 + - pkg: 40 + - pkg: 42 + - pkg: 57 + - pkg: 46 + - pkg: 47 + - pkg: 55 + - pkg: 49 + - pkg: 50 + - pkg: 52 + - pkg: 53 + - pkg: 61 + - pkg: 64 + - pkg: 65 + - pkg: 66 + - pkg: 62 + - pkg: 67 + - pkg: 68 + - pkg: 70 + - pkg: 71 + - pkg: 76 + - pkg: 77 + - pkg: 81 + - pkg: 82 + - pkg: 83 + - pkg: 5 + - pkg: 72 + - pkg: 43 + - pkg: 89 + - pkg: 100 + - pkg: 120 + - pkg: 126 + - pkg: 105 + - pkg: 119 + - pkg: 122 + - pkg: 98 + - pkg: 118 + - pkg: 87 + - pkg: 9 + - pkg: 101 + - pkg: 35 + - pkg: 44 + - pkg: 3 + - pkg: 45 + - pkg: 48 + - pkg: 99 + - pkg: 51 + - pkg: 69 + - pkg: 23 + - pkg: 59 + - pkg: 90 + - pkg: 92 + - pkg: 94 + - pkg: 56 + - pkg: 54 + - pkg: 95 + - pkg: 117 + - pkg: 125 + - {} + - pkg: 2 + - pkg: 39 + - pkg: 103 + - pkg: 108 + - pkg: 109 + - pkg: 110 + - pkg: 111 + - pkg: 112 + - pkg: 113 + - pkg: 107 + - pkg: 106 + - pkg: 30 + edges: + - from: 1 + to: 0 + - from: 5 + to: 4 + - from: 6 + to: 4 + - from: 7 + to: 3 + - from: 7 + to: 5 + - from: 7 + to: 6 + - from: 9 + to: 8 + - from: 10 + to: 8 + - from: 15 + to: 11 + - from: 15 + to: 12 + - from: 15 + to: 13 + - from: 15 + to: 14 + - from: 16 + to: 7 + - from: 16 + to: 8 + - from: 16 + to: 9 + - from: 16 + to: 10 + - from: 16 + to: 15 + - from: 21 + to: 20 + - from: 22 + to: 21 + - from: 23 + to: 22 + - from: 25 + to: 24 + - from: 27 + to: 15 + - from: 28 + to: 13 + - from: 28 + to: 15 + - from: 28 + to: 26 + - from: 28 + to: 27 + - from: 29 + to: 15 + - from: 29 + to: 25 + - from: 29 + to: 28 + - from: 32 + to: 31 + - from: 37 + to: 36 + - from: 39 + to: 15 + - from: 39 + to: 30 + - from: 39 + to: 32 + - from: 39 + to: 34 + - from: 39 + to: 37 + - from: 39 + to: 38 + - from: 41 + to: 30 + - from: 41 + to: 34 + - from: 41 + to: 39 + - from: 41 + to: 40 + - from: 42 + to: 0 + - from: 42 + to: 30 + - from: 44 + to: 43 + - from: 46 + to: 36 + - from: 46 + to: 44 + - from: 46 + to: 45 + - from: 49 + to: 0 + - from: 49 + to: 12 + - from: 49 + to: 38 + - from: 49 + to: 48 + - from: 50 + to: 49 + - from: 52 + to: 51 + - from: 56 + to: 54 + - from: 56 + to: 55 + - from: 58 + to: 57 + - from: 61 + to: 60 + - from: 64 + to: 63 + - from: 65 + to: 54 + - from: 66 + to: 33 + - from: 70 + to: 12 + - from: 70 + to: 22 + - from: 73 + to: 43 + - from: 73 + to: 72 + - from: 76 + to: 34 + - from: 76 + to: 39 + - from: 76 + to: 75 + - from: 81 + to: 15 + - from: 81 + to: 54 + - from: 81 + to: 67 + - from: 84 + to: 54 + - from: 85 + to: 43 + - from: 85 + to: 84 + - from: 89 + to: 34 + - from: 89 + to: 36 + - from: 89 + to: 66 + - from: 89 + to: 82 + - from: 89 + to: 83 + - from: 89 + to: 85 + - from: 89 + to: 86 + - from: 89 + to: 87 + - from: 89 + to: 88 + - from: 90 + to: 1 + - from: 90 + to: 30 + - from: 90 + to: 39 + - from: 90 + to: 70 + - from: 90 + to: 76 + - from: 91 + to: 77 + - from: 92 + to: 0 + - from: 92 + to: 15 + - from: 92 + to: 30 + - from: 92 + to: 32 + - from: 92 + to: 33 + - from: 92 + to: 34 + - from: 92 + to: 35 + - from: 92 + to: 37 + - from: 92 + to: 38 + - from: 92 + to: 39 + - from: 92 + to: 41 + - from: 92 + to: 42 + - from: 92 + to: 44 + - from: 92 + to: 45 + - from: 92 + to: 46 + - from: 92 + to: 47 + - from: 92 + to: 50 + - from: 92 + to: 52 + - from: 92 + to: 53 + - from: 92 + to: 56 + - from: 92 + to: 58 + - from: 92 + to: 59 + - from: 92 + to: 61 + - from: 92 + to: 62 + - from: 92 + to: 63 + - from: 92 + to: 64 + - from: 92 + to: 65 + - from: 92 + to: 66 + - from: 92 + to: 67 + - from: 92 + to: 68 + - from: 92 + to: 69 + - from: 92 + to: 70 + - from: 92 + to: 71 + - from: 92 + to: 73 + - from: 92 + to: 74 + - from: 92 + to: 75 + - from: 92 + to: 76 + - from: 92 + to: 77 + - from: 92 + to: 78 + - from: 92 + to: 79 + - from: 92 + to: 80 + - from: 92 + to: 81 + - from: 92 + to: 88 + - from: 92 + to: 89 + - from: 92 + to: 90 + - from: 92 + to: 91 + - from: 95 + to: 72 + - from: 95 + to: 94 + - from: 97 + to: 8 + - from: 97 + to: 17 + - from: 97 + to: 96 + - from: 98 + to: 60 + - from: 100 + to: 99 + - from: 102 + to: 22 + - from: 102 + to: 101 + - from: 103 + to: 58 + - from: 103 + to: 61 + - from: 103 + to: 98 + - from: 103 + to: 100 + - from: 103 + to: 101 + - from: 103 + to: 102 + - from: 107 + to: 80 + - from: 107 + to: 105 + - from: 107 + to: 106 + - from: 108 + to: 15 + - from: 111 + to: 110 + - from: 112 + to: 80 + - from: 112 + to: 111 + - from: 113 + to: 14 + - from: 113 + to: 15 + - from: 113 + to: 58 + - from: 113 + to: 103 + - from: 113 + to: 104 + - from: 113 + to: 107 + - from: 113 + to: 108 + - from: 113 + to: 109 + - from: 113 + to: 112 + - from: 125 + to: 15 + - from: 125 + to: 64 + - from: 125 + to: 72 + - from: 125 + to: 80 + - from: 125 + to: 105 + - from: 125 + to: 115 + - from: 125 + to: 116 + - from: 125 + to: 117 + - from: 125 + to: 118 + - from: 125 + to: 119 + - from: 125 + to: 120 + - from: 125 + to: 121 + - from: 125 + to: 122 + - from: 125 + to: 123 + - from: 125 + to: 124 + - from: 126 + to: 125 + - from: 127 + to: 32 + - from: 127 + to: 80 + - from: 127 + to: 125 + - from: 127 + to: 126 +scanner: null +advisor: null +evaluator: null +resolved_configuration: + package_curations: + - provider: + id: "DefaultDir" + curations: [] + - provider: + id: "DefaultFile" + curations: [] diff --git a/docs/source/conf.py b/docs/source/conf.py index 905131dc..ee17255c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,7 +6,7 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'User Guide for scanoss-py' +project = 'Documentation for scanoss-py' copyright = '2024, Scan Open Source Solutions SL' author = 'Jeronimo Ortiz' diff --git a/docs/source/index.rst b/docs/source/index.rst index ea205b54..320306f8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,10 +1,5 @@ -.. User Guide for scanoss.py documentation master file, created by - sphinx-quickstart on Fri Aug 9 17:36:25 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - ======================================= -User Guide for scanoss-py +Documentation for scanoss-py ======================================= Introduction From a3a917d992798dca3e493b0b477de18b74cc2385 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Thu, 22 Aug 2024 22:23:30 -0300 Subject: [PATCH 146/489] update --- analyzer-result.yml | 5027 ------------------------------------------- 1 file changed, 5027 deletions(-) delete mode 100644 analyzer-result.yml diff --git a/analyzer-result.yml b/analyzer-result.yml deleted file mode 100644 index 46e26d4f..00000000 --- a/analyzer-result.yml +++ /dev/null @@ -1,5027 +0,0 @@ ---- -repository: - vcs: - type: "Git" - url: "https://github.com/scanoss/scanoss.py" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/scanoss/scanoss.py.git" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "" - config: {} -analyzer: - start_time: "2024-08-17T19:07:21.548502047Z" - end_time: "2024-08-17T19:08:17.490737378Z" - environment: - ort_version: "DOCKER-SNAPSHOT" - build_jdk: "11.0.24+8" - java_version: "17.0.12" - os: "Linux" - processors: 10 - max_memory: 2055208960 - variables: - HOME: "/home/ort" - JAVA_HOME: "/opt/java/openjdk" - ANDROID_HOME: "/opt/android-sdk" - tool_versions: - NPM: "10.7.0" - config: - allow_dynamic_versions: false - skip_excluded: false - result: - projects: - - id: "NPM::tests/data/package.json:" - definition_file_path: "tests/data/package.json" - declared_licenses: [] - declared_licenses_processed: {} - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/scanoss/scanoss.py.git" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "tests/data" - homepage_url: "" - scope_names: [] - - id: "PIP::data:f5c0c3516ca0dc426f9c5411722e9632486ae362" - definition_file_path: "tests/data/requirements.txt" - declared_licenses: [] - declared_licenses_processed: {} - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/scanoss/scanoss.py.git" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "tests/data" - homepage_url: "" - scope_names: - - "install" - - id: "PIP::docs-docs:f5c0c3516ca0dc426f9c5411722e9632486ae362" - definition_file_path: "docs/requirements-docs.txt" - declared_licenses: [] - declared_licenses_processed: {} - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/scanoss/scanoss.py.git" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "docs" - homepage_url: "" - scope_names: - - "install" - - id: "PIP::requirements-dev.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362" - definition_file_path: "requirements-dev.txt" - declared_licenses: [] - declared_licenses_processed: {} - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/scanoss/scanoss.py.git" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "" - homepage_url: "" - scope_names: - - "install" - - id: "PIP::requirements-scancode.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362" - definition_file_path: "requirements-scancode.txt" - declared_licenses: [] - declared_licenses_processed: {} - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/scanoss/scanoss.py.git" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "" - homepage_url: "" - scope_names: - - "install" - - id: "PIP::requirements.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362" - definition_file_path: "requirements.txt" - declared_licenses: [] - declared_licenses_processed: {} - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/scanoss/scanoss.py.git" - revision: "f5c0c3516ca0dc426f9c5411722e9632486ae362" - path: "" - homepage_url: "" - scope_names: - - "install" - packages: - - id: "PyPI::alabaster:1.0.0" - purl: "pkg:pypi/alabaster@1.0.0" - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "A light, configurable Sphinx theme" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl" - hash: - value: "fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz" - hash: - value: "c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/sphinx-doc/alabaster.git" - revision: "" - path: "" - - id: "PyPI::attrs:24.2.0" - purl: "pkg:pypi/attrs@24.2.0" - authors: - - "Hynek Schlawack " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Classes Without Boilerplate" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/6a/21/5b6702a7f963e95456c0de2d495f67bf5fd62840ac655dc451586d23d39a/attrs-24.2.0-py3-none-any.whl" - hash: - value: "81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/fc/0f/aafca9af9315aee06a89ffde799a10a582fe8de76c563ee80bbcdc08b3fb/attrs-24.2.0.tar.gz" - hash: - value: "5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::babel:2.16.0" - purl: "pkg:pypi/babel@2.16.0" - authors: - - "Armin Ronacher " - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "Internationalization utilities" - homepage_url: "https://babel.pocoo.org/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ed/20/bc79bc575ba2e2a7f70e8a1155618bb1301eaa5132a8271373a6903f73f8/babel-2.16.0-py3-none-any.whl" - hash: - value: "368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/2a/74/f1bc80f23eeba13393b7222b11d95ca3af2c1e28edca18af487137eefed9/babel-2.16.0.tar.gz" - hash: - value: "d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/python-babel/babel.git" - revision: "" - path: "" - - id: "PyPI::backports-tarfile:1.2.0" - purl: "pkg:pypi/backports-tarfile@1.2.0" - authors: - - "\"Jason R. Coombs\" " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Backport of CPython tarfile module" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl" - hash: - value: "77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz" - hash: - value: "d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::banal:1.0.6" - purl: "pkg:pypi/banal@1.0.6" - authors: - - "Friedrich Lindenberg " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Commons of banal micro-functions for Python." - homepage_url: "http://github.com/pudo/banal" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ae/c4/7f6e6a539cc6b2da4da3b6a58d5e6f9342c870522ee46d41f8cbd2156953/banal-1.0.6-py2.py3-none-any.whl" - hash: - value: "877aacb16b17f8fa4fd29a7c44515c5a23dc1a7b26078bc41dd34829117d85e1" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/8c/a7/3301a69b4a31f7324b99332d758ae8da691f7f865ccd1b2adcd973c45344/banal-1.0.6.tar.gz" - hash: - value: "2fe02c9305f53168441948f4a03dfbfa2eacc73db30db4a93309083cb0e250a5" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pudo/banal.git" - revision: "" - path: "" - - id: "PyPI::beartype:0.18.5" - purl: "pkg:pypi/beartype@0.18.5" - authors: - - "Cecil Curry, et al. " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Unbearably fast runtime type checking in pure Python." - homepage_url: "https://beartype.readthedocs.io" - binary_artifact: - url: "https://files.pythonhosted.org/packages/64/43/7a1259741bd989723272ac7d381a43be932422abcff09a1d9f7ba212cb74/beartype-0.18.5-py3-none-any.whl" - hash: - value: "5301a14f2a9a5540fe47ec6d34d758e9cd8331d36c4760fc7a5499ab86310089" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/96/15/4e623478a9628ad4cee2391f19aba0b16c1dd6fedcb2a399f0928097b597/beartype-0.18.5.tar.gz" - hash: - value: "264ddc2f1da9ec94ff639141fbe33d22e12a9f75aa863b83b7046ffff1381927" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/beartype/beartype.git" - revision: "" - path: "" - - id: "PyPI::beautifulsoup4:4.12.3" - purl: "pkg:pypi/beautifulsoup4@4.12.3" - authors: - - "Leonard Richardson " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Screen-scraping library" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/b1/fe/e8c672695b37eecc5cbf43e1d0638d88d66ba3a44c4d321c796f4e59167f/beautifulsoup4-4.12.3-py3-none-any.whl" - hash: - value: "b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/b3/ca/824b1195773ce6166d388573fc106ce56d4a805bd7427b624e063596ec58/beautifulsoup4-4.12.3.tar.gz" - hash: - value: "74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::binaryornot:0.4.4" - purl: "pkg:pypi/binaryornot@0.4.4" - authors: - - "Audrey Roy Greenfeld " - declared_licenses: - - "BSD" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "Ultra-lightweight pure Python package to check if a file is binary\ - \ or text." - homepage_url: "https://github.com/audreyr/binaryornot" - binary_artifact: - url: "https://files.pythonhosted.org/packages/24/7e/f7b6f453e6481d1e233540262ccbfcf89adcd43606f44a028d7f5fae5eb2/binaryornot-0.4.4-py2.py3-none-any.whl" - hash: - value: "b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/a7/fe/7ebfec74d49f97fc55cd38240c7a7d08134002b1e14be8c3897c0dd5e49b/binaryornot-0.4.4.tar.gz" - hash: - value: "359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/audreyr/binaryornot.git" - revision: "" - path: "" - - id: "PyPI::boolean-py:4.0" - purl: "pkg:pypi/boolean-py@4.0" - authors: - - "Sebastian Kraemer " - declared_licenses: - - "BSD-2-Clause" - declared_licenses_processed: - spdx_expression: "BSD-2-Clause" - description: "Define boolean algebras, create and parse boolean expressions\ - \ and create custom boolean DSL." - homepage_url: "https://github.com/bastikr/boolean.py" - binary_artifact: - url: "https://files.pythonhosted.org/packages/3f/02/6389ef0529af6da0b913374dedb9bbde8eabfe45767ceec38cc37801b0bd/boolean.py-4.0-py3-none-any.whl" - hash: - value: "2876f2051d7d6394a531d82dc6eb407faa0b01a0a0b3083817ccd7323b8d96bd" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/a2/d9/b6e56a303d221fc0bdff2c775e4eef7fedd58194aa5a96fa89fb71634cc9/boolean.py-4.0.tar.gz" - hash: - value: "17b9a181630e43dde1851d42bef546d616d5d9b4480357514597e78b203d06e4" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/bastikr/boolean.py.git" - revision: "" - path: "" - - id: "PyPI::build:1.2.1" - purl: "pkg:pypi/build@1.2.1" - authors: - - "Filipe Laíns , Bernát Gábor , layday\ - \ , Henry Schreiner " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "# build" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl" - hash: - value: "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/ce/9e/2d725d2f7729c6e79ca62aeb926492abbc06e25910dd30139d60a68bcb19/build-1.2.1.tar.gz" - hash: - value: "526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::cachetools:5.4.0" - purl: "pkg:pypi/cachetools@5.4.0" - authors: - - "Thomas Kemmer " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Extensible memoizing collections and decorators" - homepage_url: "https://github.com/tkem/cachetools/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/04/e6/a1551acbaa06f3e48b311329828a34bc9c51a8cfaecdeb4d03c329a1ef85/cachetools-5.4.0-py3-none-any.whl" - hash: - value: "3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/a7/3f/ea907ec6d15f68ea7f381546ba58adcb298417a59f01a2962cb5e486489f/cachetools-5.4.0.tar.gz" - hash: - value: "b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/tkem/cachetools.git" - revision: "" - path: "" - - id: "PyPI::certifi:2024.7.4" - purl: "pkg:pypi/certifi@2024.7.4" - authors: - - "Kenneth Reitz " - declared_licenses: - - "MPL-2.0" - - "Mozilla Public License 2.0 (MPL 2.0)" - declared_licenses_processed: - spdx_expression: "MPL-2.0" - mapped: - Mozilla Public License 2.0 (MPL 2.0): "MPL-2.0" - description: "Python package for providing Mozilla's CA Bundle." - homepage_url: "https://github.com/certifi/python-certifi" - binary_artifact: - url: "https://files.pythonhosted.org/packages/1c/d5/c84e1a17bf61d4df64ca866a1c9a913874b4e9bdc131ec689a0ad013fb36/certifi-2024.7.4-py3-none-any.whl" - hash: - value: "c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/c2/02/a95f2b11e207f68bc64d7aae9666fed2e2b3f307748d5123dffb72a1bbea/certifi-2024.7.4.tar.gz" - hash: - value: "5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/certifi/python-certifi.git" - revision: "" - path: "" - - id: "PyPI::cffi:1.17.0" - purl: "pkg:pypi/cffi@1.17.0" - authors: - - "Armin Rigo, Maciej Fijalkowski " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "CFFI" - homepage_url: "http://cffi.readthedocs.org" - binary_artifact: - url: "https://files.pythonhosted.org/packages/f3/b9/f163bb3fa4fbc636ee1f2a6a4598c096cdef279823ddfaa5734e556dd206/cffi-1.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/1e/bf/82c351342972702867359cfeba5693927efe0a8dd568165490144f554b18/cffi-1.17.0.tar.gz" - hash: - value: "f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/python-cffi/cffi.git" - revision: "" - path: "" - - id: "PyPI::chardet:5.2.0" - purl: "pkg:pypi/chardet@5.2.0" - authors: - - "Mark Pilgrim " - declared_licenses: - - "GNU Lesser General Public License v2 or later (LGPLv2+)" - - "LGPL" - declared_licenses_processed: - spdx_expression: "LGPL-2.0-or-later" - mapped: - GNU Lesser General Public License v2 or later (LGPLv2+): "LGPL-2.0-or-later" - LGPL: "LGPL-2.0-or-later" - description: "Universal encoding detector for Python 3" - homepage_url: "https://github.com/chardet/chardet" - binary_artifact: - url: "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl" - hash: - value: "e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz" - hash: - value: "1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/chardet/chardet.git" - revision: "" - path: "" - - id: "PyPI::charset-normalizer:3.3.2" - purl: "pkg:pypi/charset-normalizer@3.3.2" - authors: - - "Ahmed TAHRI " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "The Real First Universal Charset Detector. Open, modern and actively\ - \ maintained alternative to Chardet." - homepage_url: "https://github.com/Ousret/charset_normalizer" - binary_artifact: - url: "https://files.pythonhosted.org/packages/40/26/f35951c45070edc957ba40a5b1db3cf60a9dbb1b350c2d5bef03e01e61de/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz" - hash: - value: "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/Ousret/charset_normalizer.git" - revision: "" - path: "" - - id: "PyPI::click:8.1.7" - purl: "pkg:pypi/click@8.1.7" - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "Composable command line interface toolkit" - homepage_url: "https://palletsprojects.com/p/click/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl" - hash: - value: "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz" - hash: - value: "ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pallets/click.git" - revision: "" - path: "" - - id: "PyPI::colorama:0.4.6" - purl: "pkg:pypi/colorama@0.4.6" - authors: - - "Jonathan Hartley " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "Cross-platform colored terminal text." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl" - hash: - value: "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz" - hash: - value: "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::commoncode:31.2.1" - purl: "pkg:pypi/commoncode@31.2.1" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - description: "Set of common utilities, originally split from ScanCode" - homepage_url: "https://github.com/nexB/commoncode" - binary_artifact: - url: "https://files.pythonhosted.org/packages/57/ab/3c3f9117bf1d0131f43e192ad336c1b821efef6551156b1d64e70535e6c0/commoncode-31.2.1-py3-none-any.whl" - hash: - value: "c1ab57f014bf92b609f95b86e5ae5961afbd7cc83cd42c2a4b9bdb3b8453fa5e" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/e8/e0/7f942e4f9e52b8c3e665707f434a96b55d276aa0d8bea4aa4dc36ad4960b/commoncode-31.2.1.tar.gz" - hash: - value: "907a75e6a64e16e19c4072c80e2406d89bde3dcebf79963d7ec6578eca22a883" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/commoncode.git" - revision: "" - path: "" - - id: "PyPI::container-inspector:33.0.0" - purl: "pkg:pypi/container-inspector@33.0.0" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - description: "Docker, containers, rootfs and virtual machine related software\ - \ composition analysis (SCA) utilities." - homepage_url: "https://github.com/nexB/container-inspector" - binary_artifact: - url: "https://files.pythonhosted.org/packages/74/aa/966e81912c7771269e6608478521a40a16460868a02fa34059b1f8be0737/container_inspector-33.0.0-py3-none-any.whl" - hash: - value: "6284ac158c7115672ab70d1b97a22b6d257b59ee12bebc76c6048585943e919f" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/57/fe/50dd7a684f740bd66ab2969888723fe6c16acff41629fb731e7e131d78bf/container_inspector-33.0.0.tar.gz" - hash: - value: "09260edb14549648da61260c1559b507e9dcb8296a6324368ba3803ca2011f7c" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/container-inspector.git" - revision: "" - path: "" - - id: "PyPI::crc32c:2.6" - purl: "pkg:pypi/crc32c@2.6" - authors: - - "The ICRAR DIA Team " - declared_licenses: - - "GNU Lesser General Public License v2 or later (LGPLv2+)" - - "LGPL-2.1-or-later" - declared_licenses_processed: - spdx_expression: "LGPL-2.0-or-later AND LGPL-2.1-or-later" - mapped: - GNU Lesser General Public License v2 or later (LGPLv2+): "LGPL-2.0-or-later" - description: "A python package implementing the crc32c algorithm in hardware\ - \ and software" - homepage_url: "https://github.com/ICRAR/crc32c" - binary_artifact: - url: "https://files.pythonhosted.org/packages/bd/95/edb7fd426f2a092e4d36377814c47d095c642710c55b1754679294bbc220/crc32c-2.6-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "eb867368bcd541933dd117074f836ce59f90ebac57df06dc1194ae669ad8f6fc" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/0d/49/d51f2ad63f7afc4a3c426c725c298ec6f74c7c00ce2609fc717fea0c533d/crc32c-2.6.tar.gz" - hash: - value: "f8c0d09e168c8af4c98fe61c772c775a2ec5d5bcc7a57f095daed423730309c8" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/ICRAR/crc32c.git" - revision: "" - path: "" - - id: "PyPI::cryptography:43.0.0" - purl: "pkg:pypi/cryptography@43.0.0" - authors: - - "The cryptography developers >" - declared_licenses: - - "Apache Software License" - - "Apache-2.0 OR BSD-3-Clause" - - "BSD License" - declared_licenses_processed: - spdx_expression: "Apache-2.0 AND (Apache-2.0 OR BSD-3-Clause)" - mapped: - Apache Software License: "Apache-2.0" - BSD License: "BSD-3-Clause" - description: "cryptography is a package which provides cryptographic recipes\ - \ and primitives to Python developers." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/77/9d/0b98c73cebfd41e4fb0439fe9ce08022e8d059f51caa7afc8934fc1edcd9/cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/69/ec/9fb9dcf4f91f0e5e76de597256c43eedefd8423aa59be95c70c4c3db426a/cryptography-43.0.0.tar.gz" - hash: - value: "b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::debian-inspector:31.1.0" - purl: "pkg:pypi/debian-inspector@31.1.0" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0 AND BSD-3-Clause AND MIT" - declared_licenses_processed: - spdx_expression: "Apache-2.0 AND BSD-3-Clause AND MIT" - description: "Utilities to parse Debian package, copyright and control files." - homepage_url: "https://github.com/nexB/debian-inspector" - binary_artifact: - url: "https://files.pythonhosted.org/packages/09/61/709907d112553b39c8c907f0f90618de58ec87ca4565959d2ad350c84b9f/debian_inspector-31.1.0-py3-none-any.whl" - hash: - value: "77dfeb34492dd49d8593d4f7146ffa3f71fca703737824e09d7472e0eafca567" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/90/7a/6f9d38aabf50c1e0449e22e42485047f9d22792664e1006b14aba8d2f604/debian_inspector-31.1.0.tar.gz" - hash: - value: "ebcfbc17064f10bd3b6d2122cdbc97b71a494af0ebbafaf9a8ceadfe8b164f99" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/debian-inspector.git" - revision: "" - path: "" - - id: "PyPI::dockerfile-parse:2.0.1" - purl: "pkg:pypi/dockerfile-parse@2.0.1" - authors: - - "Jiri Popelka " - declared_licenses: - - "BSD" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "Python library for Dockerfile manipulation" - homepage_url: "https://github.com/containerbuildsystem/dockerfile-parse" - binary_artifact: - url: "https://files.pythonhosted.org/packages/7a/6c/79cd5bc1b880d8c1a9a5550aa8dacd57353fa3bb2457227e1fb47383eb49/dockerfile_parse-2.0.1-py2.py3-none-any.whl" - hash: - value: "bdffd126d2eb26acf1066acb54cb2e336682e1d72b974a40894fac76a4df17f6" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/92/df/929ee0b5d2c8bd8d713c45e71b94ab57c7e11e322130724d54f469b2cd48/dockerfile-parse-2.0.1.tar.gz" - hash: - value: "3184ccdc513221983e503ac00e1aa504a2aa8f84e5de673c46b0b6eee99ec7bc" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/containerbuildsystem/dockerfile-parse.git" - revision: "" - path: "" - - id: "PyPI::docutils:0.21.2" - purl: "pkg:pypi/docutils@0.21.2" - authors: - - "David Goodger " - declared_licenses: - - "BSD License" - - "GNU General Public License (GPL)" - - "Public Domain" - - "Python Software Foundation License" - declared_licenses_processed: - spdx_expression: "GPL-3.0-or-later AND LicenseRef-scancode-public-domain-disclaimer\ - \ AND PSF-2.0" - mapped: - GNU General Public License (GPL): "GPL-3.0-or-later" - Public Domain: "LicenseRef-scancode-public-domain-disclaimer" - Python Software Foundation License: "PSF-2.0" - unmapped: - - "BSD License" - description: "Docutils -- Python Documentation Utilities" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl" - hash: - value: "dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz" - hash: - value: "3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::dparse2:0.7.0" - purl: "pkg:pypi/dparse2@0.7.0" - authors: - - "originally from Jannis Gebauer, maintained by AboutCode.org " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A parser for Python dependency files" - homepage_url: "https://github.com/nexB/dparse2" - binary_artifact: - url: "https://files.pythonhosted.org/packages/22/e9/a370e566f84807cff908e71a4824ae00ea8196319f4e2956e82509a5f1c6/dparse2-0.7.0-py3-none-any.whl" - hash: - value: "2b935161700cdad4f27fa7ada85900756739be65ba3ef614ac4436e7ba929102" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/dc/d2/59a42c7b40c1075d49aa6b5ea32a5baa87f8022d252ccb4762ca9d5a30f5/dparse2-0.7.0.tar.gz" - hash: - value: "6bf6872aeaffedcac67ad0abb516630bad045dbdb58505b58d8f796ee91f0a73" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/dparse2.git" - revision: "" - path: "" - - id: "PyPI::dukpy:0.4.0" - purl: "pkg:pypi/dukpy@0.4.0" - authors: - - "Alessandro Molina " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Simple JavaScript interpreter for Python" - homepage_url: "https://github.com/amol-/dukpy" - binary_artifact: - url: "https://files.pythonhosted.org/packages/b2/11/ef428e024465396d8c76a7c6ea9047bf95c33ed7070bd45e96fab164c704/dukpy-0.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "601adc77605fa83ad6f4b201fd6701528c1eee1ec7de2465ca8a23c636f39552" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d1/0b/402194ebcd92bb5a743106c0f4af8cf6fc75bcfeb441b90290accb197745/dukpy-0.4.0.tar.gz" - hash: - value: "677ec7102d1c1c511f7ef918078e8099778dbcea7caf3d6a2a2a72f72aa2d135" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/amol-/dukpy.git" - revision: "" - path: "" - - id: "PyPI::fasteners:0.19" - purl: "pkg:pypi/fasteners@0.19" - authors: - - "Joshua Harlow" - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "A python package that provides useful locks" - homepage_url: "https://github.com/harlowja/fasteners" - binary_artifact: - url: "https://files.pythonhosted.org/packages/61/bf/fd60001b3abc5222d8eaa4a204cd8c0ae78e75adc688f33ce4bf25b7fafa/fasteners-0.19-py3-none-any.whl" - hash: - value: "758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/5f/d4/e834d929be54bfadb1f3e3b931c38e956aaa3b235a46a3c764c26c774902/fasteners-0.19.tar.gz" - hash: - value: "b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/harlowja/fasteners.git" - revision: "" - path: "" - - id: "PyPI::filelock:3.15.4" - purl: "pkg:pypi/filelock@3.15.4" - declared_licenses: - - "The Unlicense (Unlicense)" - declared_licenses_processed: - spdx_expression: "Unlicense" - mapped: - The Unlicense (Unlicense): "Unlicense" - description: "A platform independent file lock." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ae/f0/48285f0262fe47103a4a45972ed2f9b93e4c80b8fd609fa98da78b2a5706/filelock-3.15.4-py3-none-any.whl" - hash: - value: "6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/08/dd/49e06f09b6645156550fb9aee9cc1e59aba7efbc972d665a1bd6ae0435d4/filelock-3.15.4.tar.gz" - hash: - value: "2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/tox-dev/py-filelock.git" - revision: "" - path: "" - - id: "PyPI::fingerprints:1.2.3" - purl: "pkg:pypi/fingerprints@1.2.3" - authors: - - "Friedrich Lindenberg " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A library to generate entity fingerprints." - homepage_url: "http://github.com/alephdata/fingerprints" - binary_artifact: - url: "https://files.pythonhosted.org/packages/7d/2b/24a2675458df250e144174b0d18d70ee031eed5c108256200a68aaf087f9/fingerprints-1.2.3-py2.py3-none-any.whl" - hash: - value: "b8f83ad13dcdadce94903383db3b9b062b85a3a86f54f9e26d8faa97313f20bf" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/cb/17/292aab0190d8c80647ad0961c3fb9830016541b3d54fa4a67b5327f4e922/fingerprints-1.2.3.tar.gz" - hash: - value: "1719f808ec8dd6c7b32c79129be3cc77dc2d2258008cd0236654862a86a78b97" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/alephdata/fingerprints.git" - revision: "" - path: "" - - id: "PyPI::ftfy:6.2.3" - purl: "pkg:pypi/ftfy@6.2.3" - authors: - - "Robyn Speer " - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Fixes mojibake and other problems with Unicode, after the fact" - homepage_url: "https://ftfy.readthedocs.io/en/latest/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ed/46/14d230ad057048aea7ccd2f96a80905830866d281ea90a6662a825490659/ftfy-6.2.3-py3-none-any.whl" - hash: - value: "f15761b023f3061a66207d33f0c0149ad40a8319fd16da91796363e2c049fdf8" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/da/a9/59f4354257e8350a25be1774021991fb3a99a2fb87d0c1f367592548aed3/ftfy-6.2.3.tar.gz" - hash: - value: "79b505988f29d577a58a9069afe75553a02a46e42de6091c0660cdc67812badc" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::furo:2024.8.6" - purl: "pkg:pypi/furo@2024.8.6" - authors: - - "Pradyun Gedam " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A clean customisable Sphinx documentation theme." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/27/48/e791a7ed487dbb9729ef32bb5d1af16693d8925f4366befef54119b2e576/furo-2024.8.6-py3-none-any.whl" - hash: - value: "6cd97c58b47813d3619e63e9081169880fbe331f0ca883c871ff1f3f11814f5c" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/a0/e2/d351d69a9a9e4badb4a5be062c2d0e87bd9e6c23b5e57337fef14bef34c8/furo-2024.8.6.tar.gz" - hash: - value: "b63e4cee8abfc3136d3bc03a3d45a76a850bada4d6374d24c1716b0e01394a01" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::gemfileparser2:0.9.3" - purl: "pkg:pypi/gemfileparser2@0.9.3" - authors: - - "Balasankar C, Rohit Potter, nexB. Inc. and others " - declared_licenses: - - "GPL-3.0-or-later OR MIT" - declared_licenses_processed: - spdx_expression: "GPL-3.0-or-later OR MIT" - description: "Parse Ruby Gemfile, .gemspec and Cocoapod .podspec files using\ - \ Python." - homepage_url: "https://github.com/nexB/gemfileparser2" - binary_artifact: - url: "https://files.pythonhosted.org/packages/c4/77/1478ebca9228029ba6df583704bc0f4bb05b8def0608f683bb09690819f9/gemfileparser2-0.9.3-py3-none-any.whl" - hash: - value: "6d19bd99a81dff98dafed4437f5194a383b4b22d6be1de2c92cb134a5a598152" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d3/39/2ad53fa5b9148632825aa28d1c7f9e96a092066779a097347e6941efcfc0/gemfileparser2-0.9.3.tar.gz" - hash: - value: "04528964e7f45b66f460d6ca2309eb9a8286bed3fc03a47d3eb52dee4602fc39" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/gemfileparser2.git" - revision: "" - path: "" - - id: "PyPI::google-api-core:2.19.1" - purl: "pkg:pypi/google-api-core@2.19.1" - authors: - - "Google LLC " - declared_licenses: - - "Apache 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "Google API client core library" - homepage_url: "https://github.com/googleapis/python-api-core" - binary_artifact: - url: "https://files.pythonhosted.org/packages/44/99/daa3541e8ecd7d8b7907b714ba92126097a976b5b3dbabdb5febdcf08554/google_api_core-2.19.1-py3-none-any.whl" - hash: - value: "f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/c2/41/42a127bf163d9bf1f21540a3bf41c69b231b88707d8d753680b8878201a6/google-api-core-2.19.1.tar.gz" - hash: - value: "f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/googleapis/python-api-core.git" - revision: "" - path: "" - - id: "PyPI::google-auth:2.33.0" - purl: "pkg:pypi/google-auth@2.33.0" - authors: - - "Google Cloud Platform " - declared_licenses: - - "Apache 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "Google Authentication Library" - homepage_url: "https://github.com/googleapis/google-auth-library-python" - binary_artifact: - url: "https://files.pythonhosted.org/packages/60/57/0f37c6f35847e26b7bea7d5e4f069cf037fd792cf8b67206311761e7bb92/google_auth-2.33.0-py2.py3-none-any.whl" - hash: - value: "8eff47d0d4a34ab6265c50a106a3362de6a9975bb08998700e389f857e4d39df" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/28/4d/626b37c6bcc1f211aef23f47c49375072c0cb19148627d98c85e099acbc8/google_auth-2.33.0.tar.gz" - hash: - value: "d6a52342160d7290e334b4d47ba390767e4438ad0d45b7630774533e82655b95" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/googleapis/google-auth-library-python.git" - revision: "" - path: "" - - id: "PyPI::googleapis-common-protos:1.63.2" - purl: "pkg:pypi/googleapis-common-protos@1.63.2" - authors: - - "Google LLC " - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Common protobufs used in Google APIs" - homepage_url: "https://github.com/googleapis/python-api-common-protos" - binary_artifact: - url: "https://files.pythonhosted.org/packages/02/48/87422ff1bddcae677fb6f58c97f5cfc613304a5e8ce2c3662760199c0a84/googleapis_common_protos-1.63.2-py2.py3-none-any.whl" - hash: - value: "27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/0b/1a/41723ae380fa9c561cbe7b61c4eef9091d5fe95486465ccfc84845877331/googleapis-common-protos-1.63.2.tar.gz" - hash: - value: "27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/googleapis/python-api-common-protos.git" - revision: "" - path: "" - - id: "PyPI::grpcio:1.65.5" - purl: "pkg:pypi/grpcio@1.65.5" - authors: - - "The gRPC Authors " - declared_licenses: - - "Apache License 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache License 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "HTTP/2-based RPC framework" - homepage_url: "https://grpc.io" - binary_artifact: - url: "https://files.pythonhosted.org/packages/99/6a/d9021f91eacf30e6410f4d1809517a950f0e8b9ccd9f1a0afa05b0d1c07c/grpcio-1.65.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "c3655139d7be213c32c79ef6fb2367cae28e56ef68e39b1961c43214b457f257" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/6c/d8/1d8f1640649808db79b689d65b03556077d5504baad5ea64b167a5adedad/grpcio-1.65.5.tar.gz" - hash: - value: "ec6f219fb5d677a522b0deaf43cea6697b16f338cb68d009e30930c4aa0d2209" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/grpc/grpc.git" - revision: "" - path: "" - - id: "PyPI::grpcio-tools:1.65.5" - purl: "pkg:pypi/grpcio-tools@1.65.5" - authors: - - "The gRPC Authors " - declared_licenses: - - "Apache License 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache License 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "Protobuf code generator for gRPC" - homepage_url: "https://grpc.io" - binary_artifact: - url: "https://files.pythonhosted.org/packages/24/8f/e25a413bfbe9d8d4008f7bec870621263e0d014be3ebcdd07538d04fe214/grpcio_tools-1.65.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "777243e4f7152da9d226d9cc1e6d7c2b94335e267c618260e6255a063bb7dfcb" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/78/2b/5979958c17f0f54fab1b3707a060d2780bd711698d1dc524b2208bfd8102/grpcio_tools-1.65.5.tar.gz" - hash: - value: "7c3a47ad0070bc907c7818caf55aa1948e9282d24e27afd21015872a25594bc7" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/grpc/grpc.git" - revision: "master" - path: "tools/distrib/python/grpcio_tools" - - id: "PyPI::html5lib:1.1" - purl: "pkg:pypi/html5lib@1.1" - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "HTML parser based on the WHATWG HTML specification" - homepage_url: "https://github.com/html5lib/html5lib-python" - binary_artifact: - url: "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl" - hash: - value: "0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz" - hash: - value: "b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/html5lib/html5lib-python.git" - revision: "" - path: "" - - id: "PyPI::idna:3.7" - purl: "pkg:pypi/idna@3.7" - authors: - - "Kim Davies " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "Internationalized Domain Names in Applications (IDNA)" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl" - hash: - value: "82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/21/ed/f86a79a07470cb07819390452f178b3bef1d375f2ec021ecfc709fc7cf07/idna-3.7.tar.gz" - hash: - value: "028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/kjd/idna.git" - revision: "" - path: "" - - id: "PyPI::imagesize:1.4.1" - purl: "pkg:pypi/imagesize@1.4.1" - authors: - - "Yoshiki Shibukawa " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Getting image size from png/jpeg/jpeg2000/gif file" - homepage_url: "https://github.com/shibukawa/imagesize_py" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl" - hash: - value: "0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz" - hash: - value: "69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/shibukawa/imagesize_py.git" - revision: "" - path: "" - - id: "PyPI::importlib-metadata:8.2.0" - purl: "pkg:pypi/importlib-metadata@8.2.0" - authors: - - "\"Jason R. Coombs\" " - declared_licenses: - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Read metadata from Python packages" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/82/47/bb25ec04985d0693da478797c3d8c1092b140f3a53ccb984fbbd38affa5b/importlib_metadata-8.2.0-py3-none-any.whl" - hash: - value: "11901fa0c2f97919b288679932bb64febaeacf289d18ac84dd68cb2e74213369" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/f6/a1/db39a513aa99ab3442010a994eef1cb977a436aded53042e69bee6959f74/importlib_metadata-8.2.0.tar.gz" - hash: - value: "72e8d4399996132204f9a16dcc751af254a48f8d1b20b9ff0f98d4a8f901e73d" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/python/importlib_metadata.git" - revision: "" - path: "" - - id: "PyPI::importlib-resources:6.4.3" - purl: "pkg:pypi/importlib-resources@6.4.3" - authors: - - "Barry Warsaw " - declared_licenses: - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Read resources from Python packages" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/bc/8b/e848c888201b211159cfceaac65cc3bc1e32ed9ab6ca30366c43e5f1969b/importlib_resources-6.4.3-py3-none-any.whl" - hash: - value: "2d6dfe3b9e055f72495c2085890837fc8c758984e209115c8792bddcb762cd93" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/61/b3/0412c28d21e31447e97728efcf8913afe1936692917629e6bdb847563484/importlib_resources-6.4.3.tar.gz" - hash: - value: "4a202b9b9d38563b46da59221d77bb73862ab5d79d461307bcb826d725448b98" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/python/importlib_resources.git" - revision: "" - path: "" - - id: "PyPI::intbitset:3.1.0" - purl: "pkg:pypi/intbitset@3.1.0" - authors: - - "Invenio collaboration, maintained by Philippe Ombredanne and AboutCode.org\ - \ " - declared_licenses: - - "GNU Lesser General Public License v3 or later (LGPLv3+)" - - "LGPL-3.0-or-later" - declared_licenses_processed: - spdx_expression: "LGPL-3.0-or-later" - mapped: - GNU Lesser General Public License v3 or later (LGPLv3+): "LGPL-3.0-or-later" - description: "C-based extension implementing fast integer bit sets." - homepage_url: "http://github.com/inveniosoftware-contrib/intbitset/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/24/70/b1ac992230c58501d31c6d6b50f61cf2ce72eb3a6890fc5df3d7999c5b80/intbitset-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "353ea6b4c2f2c0aba4bd7b92bf6116fa82b16de9255b8b11a3d799c9cc0b640e" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/17/1e/a0de4407565ae27f5c6352392f72c0a1238f9f28b07f8cd34fbc716b0bf6/intbitset-3.1.0.tar.gz" - hash: - value: "6e83c5ba7fda2520aa8565428bbaf842deb7293d665f3cd8281cb39254d2ff71" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/inveniosoftware-contrib/intbitset.git" - revision: "" - path: "" - - id: "PyPI::isodate:0.6.1" - purl: "pkg:pypi/isodate@0.6.1" - authors: - - "Gerhard Weis " - declared_licenses: - - "BSD" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "An ISO 8601 date/time/duration parser and formatter" - homepage_url: "https://github.com/gweis/isodate/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/b6/85/7882d311924cbcfc70b1890780763e36ff0b140c7e51c110fc59a532f087/isodate-0.6.1-py2.py3-none-any.whl" - hash: - value: "0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/db/7a/c0a56c7d56c7fa723988f122fa1f1ccf8c5c4ccc48efad0d214b49e5b1af/isodate-0.6.1.tar.gz" - hash: - value: "48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/gweis/isodate.git" - revision: "" - path: "" - - id: "PyPI::jaraco-classes:3.4.0" - purl: "pkg:pypi/jaraco-classes@3.4.0" - authors: - - "Jason R. Coombs " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Utility functions for Python class constructs" - homepage_url: "https://github.com/jaraco/jaraco.classes" - binary_artifact: - url: "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl" - hash: - value: "f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz" - hash: - value: "47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/jaraco/jaraco.classes.git" - revision: "" - path: "" - - id: "PyPI::jaraco-context:5.3.0" - purl: "pkg:pypi/jaraco-context@5.3.0" - authors: - - "Jason R. Coombs " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Useful decorators and context managers" - homepage_url: "https://github.com/jaraco/jaraco.context" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d2/40/11b7bc1898cf1dcb87ccbe09b39f5088634ac78bb25f3383ff541c2b40aa/jaraco.context-5.3.0-py3-none-any.whl" - hash: - value: "3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/c9/60/e83781b07f9a66d1d102a0459e5028f3a7816fdd0894cba90bee2bbbda14/jaraco.context-5.3.0.tar.gz" - hash: - value: "c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/jaraco/jaraco.context.git" - revision: "" - path: "" - - id: "PyPI::jaraco-functools:4.0.2" - purl: "pkg:pypi/jaraco-functools@4.0.2" - authors: - - "\"Jason R. Coombs\" " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Functools like those found in stdlib" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/b1/54/7623e24ffc63730c3a619101361b08860c6b7c7cfc1aef6edb66d80ed708/jaraco.functools-4.0.2-py3-none-any.whl" - hash: - value: "c9d16a3ed4ccb5a889ad8e0b7a343401ee5b2a71cee6ed192d3f68bc351e94e3" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/03/b1/6ca3c2052e584e9908a2c146f00378939b3c51b839304ab8ef4de067f042/jaraco_functools-4.0.2.tar.gz" - hash: - value: "3460c74cd0d32bf82b9576bbb3527c4364d5b27a21f5158a62aed6c4b42e23f5" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/jaraco/jaraco.functools.git" - revision: "" - path: "" - - id: "PyPI::javaproperties:0.8.1" - purl: "pkg:pypi/javaproperties@0.8.1" - authors: - - "John Thorvald Wodder II " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Read & write Java .properties files" - homepage_url: "https://github.com/jwodder/javaproperties" - binary_artifact: - url: "https://files.pythonhosted.org/packages/47/e8/c244dd03cecdebaf8116c93afaa1c72c8d4833f078a5d35e00c3d2c3be64/javaproperties-0.8.1-py3-none-any.whl" - hash: - value: "0e9b43334d6c1a9bffe34e2ece52588e21a7e099869bdaa481a5c6498774e18e" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/68/52/d7db7b671e2d4596c759fb526864837677c1562462e45f0ba46aef9a28c5/javaproperties-0.8.1.tar.gz" - hash: - value: "9dcba389effe67d3f906bbdcc64b8ef2ee8eac00072406784ea636bb6ba56061" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/jwodder/javaproperties.git" - revision: "" - path: "" - - id: "PyPI::jeepney:0.8.0" - purl: "pkg:pypi/jeepney@0.8.0" - authors: - - "Thomas Kluyver " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Low-level, pure Python DBus protocol wrapper." - homepage_url: "https://gitlab.com/takluyver/jeepney" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ae/72/2a1e2290f1ab1e06f71f3d0f1646c9e4634e70e1d37491535e19266e8dc9/jeepney-0.8.0-py3-none-any.whl" - hash: - value: "c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d6/f4/154cf374c2daf2020e05c3c6a03c91348d59b23c5366e968feb198306fdf/jeepney-0.8.0.tar.gz" - hash: - value: "5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://gitlab.com/takluyver/jeepney.git" - revision: "" - path: "" - - id: "PyPI::jinja2:3.1.4" - purl: "pkg:pypi/jinja2@3.1.4" - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "A very fast and expressive template engine." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/31/80/3a54838c3fb461f6fec263ebf3a3a41771bd05190238de3486aae8540c36/jinja2-3.1.4-py3-none-any.whl" - hash: - value: "bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/ed/55/39036716d19cab0747a5020fc7e907f362fbf48c984b14e62127f7e68e5d/jinja2-3.1.4.tar.gz" - hash: - value: "4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pallets/jinja.git" - revision: "" - path: "" - - id: "PyPI::jsonstreams:0.6.0" - purl: "pkg:pypi/jsonstreams@0.6.0" - authors: - - "Dylan Baker " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A JSON streaming writer" - homepage_url: "https://github.com/dcbaker/jsonstreams" - binary_artifact: - url: "https://files.pythonhosted.org/packages/af/be/233b55906cc033b890c2e4593077bc10c7e09257c46f5253dd9b2850f3f4/jsonstreams-0.6.0-py2.py3-none-any.whl" - hash: - value: "b2e609c2bc17eec77fe26dae4d32556ba59dafbbff30c9a4909f2e19fa5bb000" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/85/8c/01333839805428590015bb4cbc3b730876609e536954eb1140d24b410bd0/jsonstreams-0.6.0.tar.gz" - hash: - value: "721cda7391e9415b7b15cebd6cf92fc7f8788ca211eda7d64162a066ee45a72e" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/dcbaker/jsonstreams.git" - revision: "" - path: "" - - id: "PyPI::keyring:25.3.0" - purl: "pkg:pypi/keyring@25.3.0" - authors: - - "Kang Zhang " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Store and access your passwords safely." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/63/42/ea8c9726e5ee5ff0731978aaf7cd5fa16674cf549c46279b279d7167c2b4/keyring-25.3.0-py3-none-any.whl" - hash: - value: "8d963da00ccdf06e356acd9bf3b743208878751032d8599c6cc89eb51310ffae" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/32/30/bfdde7294ba6bb2f519950687471dc6a0996d4f77ab30d75c841fa4994ed/keyring-25.3.0.tar.gz" - hash: - value: "8d85a1ea5d6db8515b59e1c5d1d1678b03cf7fc8b8dcfb1651e8c4a524eb42ef" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/jaraco/keyring.git" - revision: "" - path: "" - - id: "PyPI::license-expression:30.3.1" - purl: "pkg:pypi/license-expression@30.3.1" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - description: "license-expression is a comprehensive utility library to parse,\ - \ compare, simplify and normalize license expressions (such as SPDX license\ - \ expressions) using boolean logic." - homepage_url: "https://github.com/aboutcode-org/license-expression" - binary_artifact: - url: "https://files.pythonhosted.org/packages/91/84/a7cf5dfa141501a20cb63595f02edfe38e0db2e3cc34e4f3cd273cc285df/license_expression-30.3.1-py3-none-any.whl" - hash: - value: "97904b9185c7bbb1e98799606fa7424191c375e70ba63a524b6f7100e42ddc46" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/57/8b/dbe230196eee2de208ba87dcfae69c46db9d7ed70e2f30f143bf994ee075/license_expression-30.3.1.tar.gz" - hash: - value: "60d5bec1f3364c256a92b9a08583d7ea933c7aa272c8d36d04144a89a3858c01" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/aboutcode-org/license-expression.git" - revision: "" - path: "" - - id: "PyPI::lxml:5.3.0" - purl: "pkg:pypi/lxml@5.3.0" - authors: - - "lxml dev team " - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "Powerful and Pythonic XML processing library combining libxml2/libxslt\ - \ with the ElementTree API." - homepage_url: "https://lxml.de/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/67/a4/1f5fbd3f58d4069000522196b0b776a014f3feec1796da03e495cf23532d/lxml-5.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/e7/6b/20c3a4b24751377aaa6307eb230b66701024012c29dd374999cc92983269/lxml-5.3.0.tar.gz" - hash: - value: "4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/lxml/lxml.git" - revision: "" - path: "" - - id: "PyPI::markdown-it-py:3.0.0" - purl: "pkg:pypi/markdown-it-py@3.0.0" - authors: - - "Chris Sewell " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Python port of markdown-it. Markdown parsing, done right!" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl" - hash: - value: "355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz" - hash: - value: "e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::markupsafe:2.1.5" - purl: "pkg:pypi/markupsafe@2.1.5" - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "Safely add untrusted strings to HTML/XML markup." - homepage_url: "https://palletsprojects.com/p/markupsafe/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/97/18/c30da5e7a0e7f4603abfc6780574131221d9148f323752c2755d48abad30/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/87/5b/aae44c6655f3801e81aa3eef09dbbf012431987ba564d7231722f68df02d/MarkupSafe-2.1.5.tar.gz" - hash: - value: "d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pallets/markupsafe.git" - revision: "" - path: "" - - id: "PyPI::mdurl:0.1.2" - purl: "pkg:pypi/mdurl@0.1.2" - authors: - - "Taneli Hukkinen " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Markdown URL utilities" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl" - hash: - value: "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz" - hash: - value: "bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::more-itertools:10.4.0" - purl: "pkg:pypi/more-itertools@10.4.0" - authors: - - "Erik Rose " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "More routines for operating on iterables, beyond itertools" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d8/0b/6a51175e1395774449fca317fb8861379b7a2d59be411b8cce3d19d6ce78/more_itertools-10.4.0-py3-none-any.whl" - hash: - value: "0f7d9f83a0a8dcfa8a2694a770590d98a67ea943e3d9f5298309a484758c4e27" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/92/0d/ad6a82320cb8eba710fd0dceb0f678d5a1b58d67d03ae5be14874baa39e0/more-itertools-10.4.0.tar.gz" - hash: - value: "fe0e63c4ab068eac62410ab05cccca2dc71ec44ba8ef29916a0090df061cf923" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::mutf8:1.0.6" - purl: "pkg:pypi/mutf8@1.0.6" - authors: - - "Tyler Kennedy " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Fast MUTF-8 encoder & decoder" - homepage_url: "http://github.com/TkTech/mutf8" - binary_artifact: - url: "" - hash: - value: "" - algorithm: "" - source_artifact: - url: "https://files.pythonhosted.org/packages/ca/31/3c57313757b3a47dcf32d2a9bad55d913b797efc8814db31bed8a7142396/mutf8-1.0.6.tar.gz" - hash: - value: "1bbbefb67c2e5a57104750bb04b0912200b57b2fa9841be245279e83859cb346" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/TkTech/mutf8.git" - revision: "" - path: "" - - id: "PyPI::nh3:0.2.18" - purl: "pkg:pypi/nh3@0.2.18" - authors: - - "messense >" - declared_licenses: - - "MIT" - declared_licenses_processed: - spdx_expression: "MIT" - description: "Python bindings to the ammonia HTML sanitization library." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/1b/63/6ab90d0e5225ab9780f6c9fb52254fa36b52bb7c188df9201d05b647e5e1/nh3-0.2.18-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/62/73/10df50b42ddb547a907deeb2f3c9823022580a7a47281e8eae8e003a9639/nh3-0.2.18.tar.gz" - hash: - value: "94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/messense/nh3.git" - revision: "" - path: "" - - id: "PyPI::normality:2.5.0" - purl: "pkg:pypi/normality@2.5.0" - authors: - - "Friedrich Lindenberg " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Micro-library to normalize text strings" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ae/29/cdd620678624e76de4034d1d69eb978cae4a96983dde963586f711261196/normality-2.5.0-py2.py3-none-any.whl" - hash: - value: "d9f48daf32e351e88b9e372787c1da437df9d0d818aec6e2834b02102378df62" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/e0/12/6452229afa2331de60fe93324dd9e2eb6034cb2e2faf6867419d9c51d356/normality-2.5.0.tar.gz" - hash: - value: "a55133e972b81c4a3bf8b6dc419f262f94a4fd6f636297046f74d35c93abe153" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::packageurl-python:0.15.6" - purl: "pkg:pypi/packageurl-python@0.15.6" - authors: - - "the purl authors" - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A purl aka. Package URL parser and builder" - homepage_url: "https://github.com/package-url/packageurl-python" - binary_artifact: - url: "https://files.pythonhosted.org/packages/4b/ca/b598e18eb0820a0116690a960d85625aae50dae8ba58195e254e35c2289a/packageurl_python-0.15.6-py3-none-any.whl" - hash: - value: "a40210652c89022772a6c8340d6066f7d5dc67132141e5284a4db7a27d0a8ab0" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/56/c5/c0f3ac14fd44f9b344069397fbe79aad1fd2c69220d145447c6c29cb541d/packageurl_python-0.15.6.tar.gz" - hash: - value: "cbc89afd15d5f4d05db4f1b61297e5b97a43f61f28799f6d282aff467ed2ee96" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/package-url/packageurl-python.git" - revision: "" - path: "" - - id: "PyPI::packaging:24.1" - purl: "pkg:pypi/packaging@24.1" - authors: - - "Donald Stufft " - declared_licenses: - - "Apache Software License" - - "BSD License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - unmapped: - - "BSD License" - description: "Core utilities for Python packages" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl" - hash: - value: "5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/51/65/50db4dda066951078f0a96cf12f4b9ada6e4b811516bf0262c0f4f7064d4/packaging-24.1.tar.gz" - hash: - value: "026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pypa/packaging.git" - revision: "" - path: "" - - id: "PyPI::packvers:21.5" - purl: "pkg:pypi/packvers@21.5" - authors: - - "Donald Stufft and individual contributors " - declared_licenses: - - "Apache Software License" - - "BSD License" - - "BSD-2-Clause or Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0 AND (Apache-2.0 OR BSD-2-Clause)" - mapped: - Apache Software License: "Apache-2.0" - BSD License: "BSD-2-Clause" - BSD-2-Clause or Apache-2.0: "BSD-2-Clause OR Apache-2.0" - description: "Core utilities for Python packages. Fork to support LegacyVersion" - homepage_url: "https://github.com/nexB/packvers" - binary_artifact: - url: "https://files.pythonhosted.org/packages/00/0c/a57d44f7f970ea31dfcda7ffbe8509d087c2386a28b791867a8868fc66da/packvers-21.5-py3-none-any.whl" - hash: - value: "a05e4a2b0f2eecb49d2568bfe180168a99165ab5167aa791f82266e33740ac87" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/c5/1b/dfb8654324222e996a18297b2b2b21b637cad0a23d75134cc1fd129dd7c0/packvers-21.5.tar.gz" - hash: - value: "2d2758fc09d2c325414354b8478d649f878b52c38598517fba51c8623526ca79" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/packvers.git" - revision: "" - path: "" - - id: "PyPI::parameter-expansion-patched:0.3.1" - purl: "pkg:pypi/parameter-expansion-patched@0.3.1" - authors: - - "Michael A. Smith and Philippe Ombredanne " - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Shell parameter expansion in Python. Patched by co-maintainer\ - \ for a PyPI release." - homepage_url: "https://github.com/nexB/parameter-expansion-patched" - binary_artifact: - url: "https://files.pythonhosted.org/packages/6c/9f/2eb2762808faed5218faba5559415b5bb62b39376cf9a38acc01f9786481/parameter_expansion_patched-0.3.1-py3-none-any.whl" - hash: - value: "832f04bed2a81e32d9d233cbe27448a7a22edf9a744086dbd01066c41ad0f535" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/7e/15/0c6fa115b269418a0d53d4564809afb74684d8afa417323b406be26de08b/parameter-expansion-patched-0.3.1.tar.gz" - hash: - value: "ff5dbc89fbde582f3336562d196b710771e92baa7b6d59356a14b085a0b6740b" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/parameter-expansion-patched.git" - revision: "" - path: "" - - id: "PyPI::pdfminer-six:20240706" - purl: "pkg:pypi/pdfminer-six@20240706" - authors: - - "Yusuke Shinyama + Philippe Guglielmetti " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "PDF parser and analyzer" - homepage_url: "https://github.com/pdfminer/pdfminer.six" - binary_artifact: - url: "https://files.pythonhosted.org/packages/67/7d/44d6b90e5a293d3a975cefdc4e12a932ebba814995b2a07e37e599dd27c6/pdfminer.six-20240706-py3-none-any.whl" - hash: - value: "f4f70e74174b4b3542fcb8406a210b6e2e27cd0f0b5fd04534a8cc0d8951e38c" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/e3/37/63cb918ffa21412dd5d54e32e190e69bfc340f3d6aa072ad740bec9386bb/pdfminer.six-20240706.tar.gz" - hash: - value: "c631a46d5da957a9ffe4460c5dce21e8431dabb615fee5f9f4400603a58d95a6" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pdfminer/pdfminer.six.git" - revision: "" - path: "" - - id: "PyPI::pefile:2023.2.7" - purl: "pkg:pypi/pefile@2023.2.7" - authors: - - "Ero Carrera " - declared_licenses: - - "MIT" - declared_licenses_processed: - spdx_expression: "MIT" - description: "Python PE parsing module" - homepage_url: "https://github.com/erocarrera/pefile" - binary_artifact: - url: "https://files.pythonhosted.org/packages/55/26/d0ad8b448476d0a1e8d3ea5622dc77b916db84c6aa3cb1e1c0965af948fc/pefile-2023.2.7-py3-none-any.whl" - hash: - value: "da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/78/c5/3b3c62223f72e2360737fd2a57c30e5b2adecd85e70276879609a7403334/pefile-2023.2.7.tar.gz" - hash: - value: "82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/erocarrera/pefile.git" - revision: "" - path: "" - - id: "PyPI::pip-requirements-parser:32.0.1" - purl: "pkg:pypi/pip-requirements-parser@32.0.1" - authors: - - "The pip authors, nexB. Inc. and others " - declared_licenses: - - "MIT" - declared_licenses_processed: - spdx_expression: "MIT" - description: "pip requirements parser - a mostly correct pip requirements parsing\ - \ library because it uses pip's own code." - homepage_url: "https://github.com/nexB/pip-requirements-parser" - binary_artifact: - url: "https://files.pythonhosted.org/packages/54/d0/d04f1d1e064ac901439699ee097f58688caadea42498ec9c4b4ad2ef84ab/pip_requirements_parser-32.0.1-py3-none-any.whl" - hash: - value: "4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/5e/2a/63b574101850e7f7b306ddbdb02cb294380d37948140eecd468fae392b54/pip-requirements-parser-32.0.1.tar.gz" - hash: - value: "b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/pip-requirements-parser.git" - revision: "" - path: "" - - id: "PyPI::pkginfo:1.10.0" - purl: "pkg:pypi/pkginfo@1.10.0" - authors: - - "Tres Seaver, Agendaless Consulting " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Query metadata from sdists / bdists / installed packages." - homepage_url: "https://code.launchpad.net/~tseaver/pkginfo/trunk" - binary_artifact: - url: "https://files.pythonhosted.org/packages/56/09/054aea9b7534a15ad38a363a2bd974c20646ab1582a387a95b8df1bfea1c/pkginfo-1.10.0-py3-none-any.whl" - hash: - value: "889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/2f/72/347ec5be4adc85c182ed2823d8d1c7b51e13b9a6b0c1aae59582eca652df/pkginfo-1.10.0.tar.gz" - hash: - value: "5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::pkginfo2:30.0.0" - purl: "pkg:pypi/pkginfo2@30.0.0" - authors: - - "Maintained by nexB, Inc. Authored by Tres Seaver, Agendaless Consulting " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Query metadatdata from sdists / bdists / installed packages. Safer\ - \ fork of pkginfo to avoid doing arbitrary imports and eval()" - homepage_url: "https://github.com/nexB/pkginfo2" - binary_artifact: - url: "https://files.pythonhosted.org/packages/49/01/4e506c68c9ea09c702b1eac87e6d2cda6d6633e6ed42ec1f43662e246769/pkginfo2-30.0.0-py3-none-any.whl" - hash: - value: "f1558f3ff71c99e8f362b6d079c15ef334dfce8ab2bc623a992341baeb1e7248" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/90/8d/09cc1c99a30ac14050fc4e04e549e024be83ff72a7f63e75023501baf977/pkginfo2-30.0.0.tar.gz" - hash: - value: "5e1afbeb156febb407a9b5c16b51c5b4737c529eeda2b1607e1e277cf260669c" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/pkginfo2.git" - revision: "" - path: "" - - id: "PyPI::pluggy:1.5.0" - purl: "pkg:pypi/pluggy@1.5.0" - authors: - - "Holger Krekel " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "plugin and hook calling mechanisms for python" - homepage_url: "https://github.com/pytest-dev/pluggy" - binary_artifact: - url: "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl" - hash: - value: "44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz" - hash: - value: "2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pytest-dev/pluggy.git" - revision: "" - path: "" - - id: "PyPI::plugincode:32.0.0" - purl: "pkg:pypi/plugincode@32.0.0" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - description: "plugincode is a library that provides plugin functionality for\ - \ ScanCode toolkit." - homepage_url: "https://github.com/nexB/plugincode" - binary_artifact: - url: "https://files.pythonhosted.org/packages/27/6f/d38bd65c3bcb3787d6cf944c8458e45cd38f8dea0ef7587ff2998e786595/plugincode-32.0.0-py3-none-any.whl" - hash: - value: "344bb9943fcf4d6d05669c3c61efd4093fffa6a290fba7c5c11db15f2b51305e" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/cc/0d/e934e7ae95363f9504e78b5f962efd62add3659aa9ad7c79b6f34faa81e6/plugincode-32.0.0.tar.gz" - hash: - value: "4132d93b1755271c6e226c9da2e2044ff62ebcb873b5e958d66a8ddde9f345fa" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/plugincode.git" - revision: "" - path: "" - - id: "PyPI::ply:3.11" - purl: "pkg:pypi/ply@3.11" - authors: - - "David Beazley " - declared_licenses: - - "BSD" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - description: "Python Lex & Yacc" - homepage_url: "http://www.dabeaz.com/ply/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl" - hash: - value: "096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz" - hash: - value: "00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::progress:1.6" - purl: "pkg:pypi/progress@1.6" - authors: - - "Georgios Verigakis " - declared_licenses: - - "ISC" - - "ISC License (ISCL)" - declared_licenses_processed: - spdx_expression: "ISC" - mapped: - ISC License (ISCL): "ISC" - description: "Easy to use progress bars" - homepage_url: "http://github.com/verigak/progress/" - binary_artifact: - url: "" - hash: - value: "" - algorithm: "" - source_artifact: - url: "https://files.pythonhosted.org/packages/2a/68/d8412d1e0d70edf9791cbac5426dc859f4649afc22f2abbeb0d947cf70fd/progress-1.6.tar.gz" - hash: - value: "c9c86e98b5c03fa1fe11e3b67c1feda4788b8d0fe7336c2ff7d5644ccfba34cd" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/verigak/progress.git" - revision: "" - path: "" - - id: "PyPI::proto-plus:1.24.0" - purl: "pkg:pypi/proto-plus@1.24.0" - authors: - - "Google LLC " - declared_licenses: - - "Apache 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "Proto Plus for Python" - homepage_url: "https://github.com/googleapis/proto-plus-python.git" - binary_artifact: - url: "https://files.pythonhosted.org/packages/7c/6f/db31f0711c0402aa477257205ce7d29e86a75cb52cd19f7afb585f75cda0/proto_plus-1.24.0-py3-none-any.whl" - hash: - value: "402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/3e/fc/e9a65cd52c1330d8d23af6013651a0bc50b6d76bcbdf91fae7cd19c68f29/proto-plus-1.24.0.tar.gz" - hash: - value: "30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/googleapis/proto-plus-python.git" - revision: "" - path: "" - - id: "PyPI::protobuf:5.27.3" - purl: "pkg:pypi/protobuf@5.27.3" - authors: - - "protobuf@googlegroups.com " - declared_licenses: - - "3-Clause BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - "3-Clause BSD License": "BSD-3-Clause" - description: "" - homepage_url: "https://developers.google.com/protocol-buffers/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/e1/94/d77bd282d3d53155147166c2bbd156f540009b0d7be24330f76286668b90/protobuf-5.27.3-py3-none-any.whl" - hash: - value: "8572c6533e544ebf6899c360e91d6bcbbee2549251643d32c52cf8a5de295ba5" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/1b/61/0671db2ab2aee7c92d6c1b617c39b30a4cd973950118da56d77e7f397a9d/protobuf-5.27.3.tar.gz" - hash: - value: "82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::publicsuffix2:2.20191221" - purl: "pkg:pypi/publicsuffix2@2.20191221" - authors: - - "nexB Inc., Tomaz Solc, David Wilson and others. " - declared_licenses: - - "MIT License" - - "MIT and MPL-2.0" - - "Mozilla Public License 2.0 (MPL 2.0)" - declared_licenses_processed: - spdx_expression: "MIT AND MIT AND MPL-2.0 AND MPL-2.0" - mapped: - MIT License: "MIT" - MIT and MPL-2.0: "MIT AND MPL-2.0" - Mozilla Public License 2.0 (MPL 2.0): "MPL-2.0" - description: "Get a public suffix for a domain name using the Public Suffix\ - \ List. Forked from and using the same API as the publicsuffix package." - homepage_url: "https://github.com/nexb/python-publicsuffix2" - binary_artifact: - url: "https://files.pythonhosted.org/packages/9d/16/053c2945c5e3aebeefb4ccd5c5e7639e38bc30ad1bdc7ce86c6d01707726/publicsuffix2-2.20191221-py2.py3-none-any.whl" - hash: - value: "786b5e36205b88758bd3518725ec8cfe7a8173f5269354641f581c6b80a99893" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/5a/04/1759906c4c5b67b2903f546de234a824d4028ef24eb0b1122daa43376c20/publicsuffix2-2.20191221.tar.gz" - hash: - value: "00f8cc31aa8d0d5592a5ced19cccba7de428ebca985db26ac852d920ddd6fe7b" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexb/python-publicsuffix2.git" - revision: "" - path: "" - - id: "PyPI::pyahocorasick:2.1.0" - purl: "pkg:pypi/pyahocorasick@2.1.0" - authors: - - "Wojciech Muła " - declared_licenses: - - "BSD License" - - "BSD-3-Clause and Public-Domain" - declared_licenses_processed: - unmapped: - - "BSD License" - - "BSD-3-Clause and Public-Domain" - description: "pyahocorasick is a fast and memory efficient library for exact\ - \ or approximate multi-pattern string search. With the ``ahocorasick.Automaton``\ - \ class, you can find multiple key string occurrences at once in some input\ - \ text. You can use it as a plain dict-like Trie or convert a Trie to an\ - \ automaton for efficient Aho-Corasick search. And pickle to disk for easy\ - \ reuse of large automatons. Implemented in C and tested on Python 3.6+. Works\ - \ on Linux, macOS and Windows. BSD-3-Cause license." - homepage_url: "http://github.com/WojciechMula/pyahocorasick" - binary_artifact: - url: "https://files.pythonhosted.org/packages/31/32/17ab57fe5abcf09d2f1ceb502143447be00658761d167118441e19a2b2c6/pyahocorasick-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "a9f2728ac77bab807ba65c6ef41be30358ef0c9bb6960c9fe070d43f7024cb91" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/06/2e/075c667c27ecf2c3ed6bf3c62649625cf1e7de7fd349f63b49b794460b71/pyahocorasick-2.1.0.tar.gz" - hash: - value: "4df4845c1149e9fa4aa33f0f0aa35f5a42957a43a3d6e447c9b44e679e2672ea" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/WojciechMula/pyahocorasick.git" - revision: "" - path: "" - - id: "PyPI::pyasn1:0.6.0" - purl: "pkg:pypi/pyasn1@0.6.0" - authors: - - "Ilya Etingof " - declared_licenses: - - "BSD License" - - "BSD-2-Clause" - declared_licenses_processed: - spdx_expression: "BSD-2-Clause" - mapped: - BSD License: "BSD-2-Clause" - description: "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs\ - \ (X.208)" - homepage_url: "https://github.com/pyasn1/pyasn1" - binary_artifact: - url: "https://files.pythonhosted.org/packages/23/7e/5f50d07d5e70a2addbccd90ac2950f81d1edd0783630651d9268d7f1db49/pyasn1-0.6.0-py2.py3-none-any.whl" - hash: - value: "cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/4a/a3/d2157f333900747f20984553aca98008b6dc843eb62f3a36030140ccec0d/pyasn1-0.6.0.tar.gz" - hash: - value: "3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pyasn1/pyasn1.git" - revision: "" - path: "" - - id: "PyPI::pyasn1-modules:0.4.0" - purl: "pkg:pypi/pyasn1-modules@0.4.0" - authors: - - "Ilya Etingof " - declared_licenses: - - "BSD" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "A collection of ASN.1-based protocols modules" - homepage_url: "https://github.com/pyasn1/pyasn1-modules" - binary_artifact: - url: "https://files.pythonhosted.org/packages/13/68/8906226b15ef38e71dc926c321d2fe99de8048e9098b5dfd38343011c886/pyasn1_modules-0.4.0-py3-none-any.whl" - hash: - value: "be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/f7/00/e7bd1dec10667e3f2be602686537969a7ac92b0a7c5165be2e5875dc3971/pyasn1_modules-0.4.0.tar.gz" - hash: - value: "831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pyasn1/pyasn1-modules.git" - revision: "" - path: "" - - id: "PyPI::pycparser:2.22" - purl: "pkg:pypi/pycparser@2.22" - authors: - - "Eli Bendersky " - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "C parser in Python" - homepage_url: "https://github.com/eliben/pycparser" - binary_artifact: - url: "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl" - hash: - value: "c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz" - hash: - value: "491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/eliben/pycparser.git" - revision: "" - path: "" - - id: "PyPI::pygmars:0.8.1" - purl: "pkg:pypi/pygmars@0.8.1" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - description: "Craft simple regex-based small language lexers and parsers. Build\ - \ parsers from grammars and accept Pygments lexers as an input. Derived from\ - \ NLTK." - homepage_url: "https://github.com/aboutcode-org/pygmars" - binary_artifact: - url: "https://files.pythonhosted.org/packages/74/4e/74766da813bf2c1388277be438361dbb411210684e320788a93e02a94218/pygmars-0.8.1-py3-none-any.whl" - hash: - value: "eda9534289500c19cc8ae318839683a8c917834299a2ba4ee8777588b73d509b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/47/e2/53c68dd20a22057272acb5a71b87cf0c80fb112e2dc3bd8b25d08825d050/pygmars-0.8.1.tar.gz" - hash: - value: "07b324a7d8702ba3aec2b0243c5fec0ed7986e4c7e6926098637ee6ee894dc2d" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/aboutcode-org/pygmars.git" - revision: "" - path: "" - - id: "PyPI::pygments:2.18.0" - purl: "pkg:pypi/pygments@2.18.0" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD License" - - "BSD-2-Clause" - declared_licenses_processed: - spdx_expression: "BSD-2-Clause" - mapped: - BSD License: "BSD-2-Clause" - description: "Pygments" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl" - hash: - value: "b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/8e/62/8336eff65bcbc8e4cb5d05b55faf041285951b6e80f33e2bff2024788f31/pygments-2.18.0.tar.gz" - hash: - value: "786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pygments/pygments.git" - revision: "" - path: "" - - id: "PyPI::pymaven-patch:0.3.2" - purl: "pkg:pypi/pymaven-patch@0.3.2" - authors: - - "Walter Scheper " - declared_licenses: - - "Apache-2" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache-2: "Apache-2.0" - description: "Python access to maven. nexB advanced patch." - homepage_url: "https://github.com/nexB/pymaven" - binary_artifact: - url: "https://files.pythonhosted.org/packages/39/9a/9e597fcd70da0c2e34a9b3c60df62f23a6972295caa68f6d482b57db937b/pymaven_patch-0.3.2-py3-none-any.whl" - hash: - value: "29a67d508e5d7a55c4359435e009ab87217ceb604a48caeb7b5b7d26b3099f65" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/f2/5a/5de519f07057838d4768ed7f04ceea0629b373e06aa96bddfa7bb8d78654/pymaven-patch-0.3.2.tar.gz" - hash: - value: "0cf7c93e89f01f0408eb656eec58cb4a228c95e03b3d47cb73d31f899055cd50" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/pymaven.git" - revision: "" - path: "" - - id: "PyPI::pyopenssl:24.2.1" - purl: "pkg:pypi/pyopenssl@24.2.1" - authors: - - "The pyOpenSSL developers " - declared_licenses: - - "Apache License, Version 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache License, Version 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "Python wrapper module around the OpenSSL library" - homepage_url: "https://pyopenssl.org/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d9/dd/e0aa7ebef5168c75b772eda64978c597a9129b46be17779054652a7999e4/pyOpenSSL-24.2.1-py3-none-any.whl" - hash: - value: "967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/5d/70/ff56a63248562e77c0c8ee4aefc3224258f1856977e0c1472672b62dadb8/pyopenssl-24.2.1.tar.gz" - hash: - value: "4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pyca/pyopenssl.git" - revision: "" - path: "" - - id: "PyPI::pypac:0.16.4" - purl: "pkg:pypi/pypac@0.16.4" - authors: - - "Carson Lam <46059+carsonyl@users.noreply.github.com>" - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Proxy auto-config and auto-discovery for Python." - homepage_url: "https://github.com/carsonyl/pypac" - binary_artifact: - url: "https://files.pythonhosted.org/packages/a0/af/9d71907e51ee270f19f33210cfbc5b7bebc8ae900beecac2685f4cd4c3b5/PyPAC-0.16.4-py2.py3-none-any.whl" - hash: - value: "dc2b775c7a2c9c77b1351681fec729788b08b7c76e6d2a041fe35cf60ca493c6" - algorithm: "SHA-256" - source_artifact: - url: "" - hash: - value: "" - algorithm: "" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/carsonyl/pypac.git" - revision: "" - path: "" - - id: "PyPI::pyparsing:3.1.2" - purl: "pkg:pypi/pyparsing@3.1.2" - authors: - - "Paul McGuire " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "pyparsing module - Classes and methods to define and execute parsing\ - \ grammars" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/9d/ea/6d76df31432a0e6fdf81681a895f009a4bb47b3c39036db3e1b528191d52/pyparsing-3.1.2-py3-none-any.whl" - hash: - value: "f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/46/3a/31fd28064d016a2182584d579e033ec95b809d8e220e74c4af6f0f2e8842/pyparsing-3.1.2.tar.gz" - hash: - value: "a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::pyproject-hooks:1.1.0" - purl: "pkg:pypi/pyproject-hooks@1.1.0" - authors: - - "Thomas Kluyver " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Wrappers to call pyproject.toml-based build backend hooks." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl" - hash: - value: "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/c7/07/6f63dda440d4abb191b91dc383b472dae3dd9f37e4c1e4a5c3db150531c6/pyproject_hooks-1.1.0.tar.gz" - hash: - value: "4b37730834edbd6bd37f26ece6b44802fb1c1ee2ece0e54ddff8bfc06db86965" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pypa/pyproject-hooks.git" - revision: "" - path: "" - - id: "PyPI::pyyaml:6.0.2" - purl: "pkg:pypi/pyyaml@6.0.2" - authors: - - "Kirill Simonov " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "YAML parser and emitter for Python" - homepage_url: "https://pyyaml.org/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - hash: - value: "3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz" - hash: - value: "d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/yaml/pyyaml.git" - revision: "" - path: "" - - id: "PyPI::rdflib:7.0.0" - purl: "pkg:pypi/rdflib@7.0.0" - authors: - - "Daniel 'eikeon' Krech " - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "RDFLib is a Python library for working with RDF, a simple yet\ - \ powerful language for representing information." - homepage_url: "https://github.com/RDFLib/rdflib" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d4/b0/7b7d8b5b0d01f1a0b12cc2e5038a868ef3a15825731b8a0d776cf47566c0/rdflib-7.0.0-py3-none-any.whl" - hash: - value: "0438920912a642c866a513de6fe8a0001bd86ef975057d6962c79ce4771687cd" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/0d/a3/63740490a392921a611cfc05b5b17bffd4259b3c9589c7904a4033b3d291/rdflib-7.0.0.tar.gz" - hash: - value: "9995eb8569428059b8c1affd26b25eac510d64f5043d9ce8c84e0d0036e995ae" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/RDFLib/rdflib.git" - revision: "" - path: "" - - id: "PyPI::readme-renderer:44.0" - purl: "pkg:pypi/readme-renderer@44.0" - authors: - - "The Python Packaging Authority " - declared_licenses: - - "Apache License, Version 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache License, Version 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "readme_renderer is a library for rendering readme descriptions\ - \ for Warehouse" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/e1/67/921ec3024056483db83953ae8e48079ad62b92db7880013ca77632921dd0/readme_renderer-44.0-py3-none-any.whl" - hash: - value: "2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/5a/a9/104ec9234c8448c4379768221ea6df01260cd6c2ce13182d4eac531c8342/readme_renderer-44.0.tar.gz" - hash: - value: "8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::requests:2.32.3" - purl: "pkg:pypi/requests@2.32.3" - authors: - - "Kenneth Reitz " - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Python HTTP for Humans." - homepage_url: "https://requests.readthedocs.io" - binary_artifact: - url: "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl" - hash: - value: "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz" - hash: - value: "55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/psf/requests.git" - revision: "" - path: "" - - id: "PyPI::requests-file:2.1.0" - purl: "pkg:pypi/requests-file@2.1.0" - authors: - - "David Shea " - declared_licenses: - - "Apache 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "File transport adapter for Requests" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d7/25/dd878a121fcfdf38f52850f11c512e13ec87c2ea72385933818e5b6c15ce/requests_file-2.1.0-py2.py3-none-any.whl" - hash: - value: "cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/72/97/bf44e6c6bd8ddbb99943baf7ba8b1a8485bcd2fe0e55e5708d7fee4ff1ae/requests_file-2.1.0.tar.gz" - hash: - value: "0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::requests-toolbelt:1.0.0" - purl: "pkg:pypi/requests-toolbelt@1.0.0" - authors: - - "Ian Cordasco, Cory Benfield " - declared_licenses: - - "Apache 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "A utility belt for advanced users of python-requests" - homepage_url: "https://toolbelt.readthedocs.io/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl" - hash: - value: "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz" - hash: - value: "7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/requests/toolbelt.git" - revision: "" - path: "" - - id: "PyPI::rfc3986:2.0.0" - purl: "pkg:pypi/rfc3986@2.0.0" - authors: - - "Ian Stapleton Cordasco " - declared_licenses: - - "Apache 2.0" - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache 2.0: "Apache-2.0" - Apache Software License: "Apache-2.0" - description: "Validating URI References per RFC 3986" - homepage_url: "http://rfc3986.readthedocs.io" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl" - hash: - value: "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz" - hash: - value: "97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::rich:13.7.1" - purl: "pkg:pypi/rich@13.7.1" - authors: - - "Will McGugan " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Render rich text, tables, progress bars, syntax highlighting,\ - \ markdown and more to the terminal" - homepage_url: "https://github.com/Textualize/rich" - binary_artifact: - url: "https://files.pythonhosted.org/packages/87/67/a37f6214d0e9fe57f6ae54b2956d550ca8365857f42a1ce0392bb21d9410/rich-13.7.1-py3-none-any.whl" - hash: - value: "4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/b3/01/c954e134dc440ab5f96952fe52b4fdc64225530320a910473c1fe270d9aa/rich-13.7.1.tar.gz" - hash: - value: "9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/Textualize/rich.git" - revision: "" - path: "" - - id: "PyPI::rsa:4.9" - purl: "pkg:pypi/rsa@4.9" - authors: - - "Sybren A. Stüvel " - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Pure-Python RSA implementation" - homepage_url: "https://stuvel.eu/rsa" - binary_artifact: - url: "https://files.pythonhosted.org/packages/49/97/fa78e3d2f65c02c8e1268b9aba606569fe97f6c8f7c2d74394553347c145/rsa-4.9-py3-none-any.whl" - hash: - value: "90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/aa/65/7d973b89c4d2351d7fb232c2e452547ddfa243e93131e7cfa766da627b52/rsa-4.9.tar.gz" - hash: - value: "e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::saneyaml:0.6.1" - purl: "pkg:pypi/saneyaml@0.6.1" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - description: "Read and write readable YAML safely preserving order and avoiding\ - \ bad surprises with unwanted infered type conversions. This library is a\ - \ PyYaml wrapper with sane behaviour to read and write readable YAML safely,\ - \ typically when used for configuration." - homepage_url: "https://github.com/aboutcode-org/saneyaml" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ea/c0/b41733920cef3d87ee7d1fd5a618c7bb5240ba80dd2f29c73ec3416b3e04/saneyaml-0.6.1-py3-none-any.whl" - hash: - value: "60553363ac55433cef2bc1d6c5a1c9f6e2787e5f40e8c6fad5983eb701592c5b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/18/bb/b3ab128fe13964fc8da25ecbac82f9ed9beb59b2e04bfbef433886f1acb0/saneyaml-0.6.1.tar.gz" - hash: - value: "19cfbd8bf94d730998162c790fe5cec9abb5300cc5890fe37dc6dbcaa8fb16bb" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/aboutcode-org/saneyaml.git" - revision: "" - path: "" - - id: "PyPI::scancode-toolkit-mini:32.2.1" - purl: "pkg:pypi/scancode-toolkit-mini@32.2.1" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft" - declared_licenses_processed: - spdx_expression: "Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-copyleft\ - \ AND LicenseRef-scancode-other-permissive" - description: "ScanCode is a tool to scan code for license, copyright, package\ - \ and their documented dependencies and other interesting facts. scancode-toolkit-mini\ - \ is a special build that does not come with pre-built binary dependencies\ - \ by default. These are instead installed separately or with the extra_requires\ - \ scancode-toolkit-mini[full]" - homepage_url: "https://github.com/nexB/scancode-toolkit" - binary_artifact: - url: "https://files.pythonhosted.org/packages/29/72/5f6cce8d3a9a503a658535b22dbcd41c8cd2f59a625da1ffd0e2b2a6f98f/scancode_toolkit_mini-32.2.1-cp311-none-any.whl" - hash: - value: "d6d791d75f797107fd19b97e48754376b5d7ab8d847a4b2e5a8538f9c578b7ad" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/52/21/2e8e21c0413212d4c5457b1d1811691dc344b38b62429e995cf5f8e3218d/scancode-toolkit-mini-32.2.1.tar.gz" - hash: - value: "2a93e90b0797696bb2247fe8230e155d3fb0124408426b885b35bbf52854a00e" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/scancode-toolkit.git" - revision: "" - path: "" - - id: "PyPI::secretstorage:3.3.3" - purl: "pkg:pypi/secretstorage@3.3.3" - authors: - - "Dmitry Shachnev " - declared_licenses: - - "BSD 3-Clause License" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD 3-Clause License: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "Python bindings to FreeDesktop.org Secret Service API" - homepage_url: "https://github.com/mitya57/secretstorage" - binary_artifact: - url: "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl" - hash: - value: "f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/53/a4/f48c9d79cb507ed1373477dbceaba7401fd8a23af63b837fa61f1dcd3691/SecretStorage-3.3.3.tar.gz" - hash: - value: "2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/mitya57/secretstorage.git" - revision: "" - path: "" - - id: "PyPI::semantic-version:2.10.0" - purl: "pkg:pypi/semantic-version@2.10.0" - authors: - - "Raphaël Barrois " - declared_licenses: - - "BSD" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "A library implementing the 'SemVer' scheme." - homepage_url: "https://github.com/rbarrois/python-semanticversion" - binary_artifact: - url: "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl" - hash: - value: "de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/7d/31/f2289ce78b9b473d582568c234e104d2a342fd658cc288a7553d83bb8595/semantic_version-2.10.0.tar.gz" - hash: - value: "bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/rbarrois/python-semanticversion.git" - revision: "" - path: "" - - id: "PyPI::setuptools:72.2.0" - purl: "pkg:pypi/setuptools@72.2.0" - authors: - - "Python Packaging Authority " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Easily download, build, install, upgrade, and uninstall Python\ - \ packages" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/6e/ec/06715d912351edc453e37f93f3fc80dcffd5ca0e70386c87529aca296f05/setuptools-72.2.0-py3-none-any.whl" - hash: - value: "f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/ce/ef/013ded5b0d259f3fa636bf35de186f0061c09fbe124020ce6b8db68c83af/setuptools-72.2.0.tar.gz" - hash: - value: "80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pypa/setuptools.git" - revision: "" - path: "" - - id: "PyPI::six:1.16.0" - purl: "pkg:pypi/six@1.16.0" - authors: - - "Benjamin Peterson " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Python 2 and 3 compatibility utilities" - homepage_url: "https://github.com/benjaminp/six" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" - hash: - value: "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz" - hash: - value: "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/benjaminp/six.git" - revision: "" - path: "" - - id: "PyPI::snowballstemmer:2.2.0" - purl: "pkg:pypi/snowballstemmer@2.2.0" - authors: - - "Snowball Developers " - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "This package provides 29 stemmers for 28 languages generated from\ - \ Snowball algorithms." - homepage_url: "https://github.com/snowballstem/snowball" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl" - hash: - value: "c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/44/7b/af302bebf22c749c56c9c3e8ae13190b5b5db37a33d9068652e8f73b7089/snowballstemmer-2.2.0.tar.gz" - hash: - value: "09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/snowballstem/snowball.git" - revision: "" - path: "" - - id: "PyPI::soupsieve:2.6" - purl: "pkg:pypi/soupsieve@2.6" - authors: - - "Isaac Muse " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A modern CSS selector implementation for Beautiful Soup." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/d1/c2/fe97d779f3ef3b15f05c94a2f1e3d21732574ed441687474db9d342a7315/soupsieve-2.6-py3-none-any.whl" - hash: - value: "e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d7/ce/fbaeed4f9fb8b2daa961f90591662df6a86c1abf25c548329a86920aedfb/soupsieve-2.6.tar.gz" - hash: - value: "e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::spdx-tools:0.8.2" - purl: "pkg:pypi/spdx-tools@0.8.2" - authors: - - "\"Ahmed H. Ismail\" " - declared_licenses: - - "Apache Software License" - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "SPDX parser and tools." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/a2/a3/5d311af9214f65b0e7106f13d14b677563533f81b4953578023051f4f916/spdx_tools-0.8.2-py3-none-any.whl" - hash: - value: "8c336c873f9caaf110693a1d38c007031e67bea53aa4b881007b680be66de934" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/32/d8/a67445be5981469fdbaf7f765f53c920f699e7e512cc931b650a935c3199/spdx-tools-0.8.2.tar.gz" - hash: - value: "aea4ac9c2c375e7f439b1cef5ff32ef34914c083de0f61e08ed67cd3d9deb2a9" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::sphinx:8.0.2" - purl: "pkg:pypi/sphinx@8.0.2" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "Python documentation generator" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/4d/61/2ad169c6ff1226b46e50da0e44671592dbc6d840a52034a0193a99b28579/sphinx-8.0.2-py3-none-any.whl" - hash: - value: "56173572ae6c1b9a38911786e206a110c9749116745873feae4f9ce88e59391d" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/25/a7/3cc3d6dcad70aba2e32a3ae8de5a90026a0a2fdaaa0756925e3a120249b6/sphinx-8.0.2.tar.gz" - hash: - value: "0cce1ddcc4fd3532cf1dd283bc7d886758362c5c1de6598696579ce96d8ffa5b" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/sphinx-doc/sphinx.git" - revision: "" - path: "" - - id: "PyPI::sphinx-basic-ng:1.0.0b2" - purl: "pkg:pypi/sphinx-basic-ng@1.0.0b2" - authors: - - "Pradyun Gedam " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A modern skeleton for Sphinx themes." - homepage_url: "https://github.com/pradyunsg/sphinx-basic-ng" - binary_artifact: - url: "https://files.pythonhosted.org/packages/3c/dd/018ce05c532a22007ac58d4f45232514cd9d6dd0ee1dc374e309db830983/sphinx_basic_ng-1.0.0b2-py3-none-any.whl" - hash: - value: "eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz" - hash: - value: "9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pradyunsg/sphinx-basic-ng.git" - revision: "" - path: "" - - id: "PyPI::sphinxcontrib-applehelp:2.0.0" - purl: "pkg:pypi/sphinxcontrib-applehelp@2.0.0" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple\ - \ help books" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl" - hash: - value: "4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz" - hash: - value: "2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/sphinx-doc/sphinxcontrib-applehelp.git" - revision: "" - path: "" - - id: "PyPI::sphinxcontrib-devhelp:2.0.0" - purl: "pkg:pypi/sphinxcontrib-devhelp@2.0.0" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp\ - \ documents" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl" - hash: - value: "aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz" - hash: - value: "411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/sphinx-doc/sphinxcontrib-devhelp.git" - revision: "" - path: "" - - id: "PyPI::sphinxcontrib-htmlhelp:2.1.0" - purl: "pkg:pypi/sphinxcontrib-htmlhelp@2.1.0" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "======================" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl" - hash: - value: "166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz" - hash: - value: "c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/sphinx-doc/sphinxcontrib-htmlhelp.git" - revision: "" - path: "" - - id: "PyPI::sphinxcontrib-jsmath:1.0.1" - purl: "pkg:pypi/sphinxcontrib-jsmath@1.0.1" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "A sphinx extension which renders display math in HTML via JavaScript" - homepage_url: "http://sphinx-doc.org/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl" - hash: - value: "2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz" - hash: - value: "a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::sphinxcontrib-qthelp:2.0.0" - purl: "pkg:pypi/sphinxcontrib-qthelp@2.0.0" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp\ - \ documents" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl" - hash: - value: "b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz" - hash: - value: "4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/sphinx-doc/sphinxcontrib-qthelp.git" - revision: "" - path: "" - - id: "PyPI::sphinxcontrib-serializinghtml:2.0.0" - purl: "pkg:pypi/sphinxcontrib-serializinghtml@2.0.0" - authors: - - "Georg Brandl " - declared_licenses: - - "BSD License" - declared_licenses_processed: - unmapped: - - "BSD License" - description: "sphinxcontrib-serializinghtml is a sphinx extension which outputs\ - \ \"serialized\" HTML files (json and pickle)" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl" - hash: - value: "6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz" - hash: - value: "e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/sphinx-doc/sphinxcontrib-serializinghtml.git" - revision: "" - path: "" - - id: "PyPI::text-unidecode:1.3" - purl: "pkg:pypi/text-unidecode@1.3" - authors: - - "Mikhail Korobov " - declared_licenses: - - "Artistic License" - - "GNU General Public License (GPL)" - - "GNU General Public License v2 or later (GPLv2+)" - declared_licenses_processed: - spdx_expression: "Artistic-2.0 AND GPL-2.0-or-later AND GPL-3.0-or-later" - mapped: - Artistic License: "Artistic-2.0" - GNU General Public License (GPL): "GPL-3.0-or-later" - GNU General Public License v2 or later (GPLv2+): "GPL-2.0-or-later" - description: "The most basic Text::Unidecode port" - homepage_url: "https://github.com/kmike/text-unidecode/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl" - hash: - value: "1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz" - hash: - value: "bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/kmike/text-unidecode.git" - revision: "" - path: "" - - id: "PyPI::tldextract:5.1.2" - purl: "pkg:pypi/tldextract@5.1.2" - authors: - - "John Kurkowski " - declared_licenses: - - "BSD License" - - "BSD-3-Clause" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD License: "BSD-3-Clause" - description: "Accurately separates a URL's subdomain, domain, and public suffix,\ - \ using the Public Suffix List (PSL). By default, this includes the public\ - \ ICANN TLDs and their exceptions. You can optionally support the Public Suffix\ - \ List's private domains as well." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/fc/6d/8eaafb735b39c4ab3bb8fe4324ef8f0f0af27a7df9bb4cd503927bd5475d/tldextract-5.1.2-py3-none-any.whl" - hash: - value: "4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/db/ed/c92a5d6edaafec52f388c2d2946b4664294299cebf52bb1ef9cbc44ae739/tldextract-5.1.2.tar.gz" - hash: - value: "c9e17f756f05afb5abac04fe8f766e7e70f9fe387adb1859f0f52408ee060200" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" - - id: "PyPI::toml:0.10.2" - purl: "pkg:pypi/toml@0.10.2" - authors: - - "William Pearson " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Python Library for Tom's Obvious, Minimal Language" - homepage_url: "https://github.com/uiri/toml" - binary_artifact: - url: "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl" - hash: - value: "806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz" - hash: - value: "b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/uiri/toml.git" - revision: "" - path: "" - - id: "PyPI::twine:5.1.1" - purl: "pkg:pypi/twine@5.1.1" - authors: - - "Donald Stufft and individual contributors " - declared_licenses: - - "Apache Software License" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - mapped: - Apache Software License: "Apache-2.0" - description: "Collection of utilities for publishing packages on PyPI" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/5d/ec/00f9d5fd040ae29867355e559a94e9a8429225a0284a3f5f091a3878bfc0/twine-5.1.1-py3-none-any.whl" - hash: - value: "215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/77/68/bd982e5e949ef8334e6f7dcf76ae40922a8750aa2e347291ae1477a4782b/twine-5.1.1.tar.gz" - hash: - value: "9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pypa/twine.git" - revision: "" - path: "" - - id: "PyPI::typecode:30.0.2" - purl: "pkg:pypi/typecode@30.0.2" - authors: - - "nexB. Inc. and others " - declared_licenses: - - "Apache-2.0" - declared_licenses_processed: - spdx_expression: "Apache-2.0" - description: "Comprehensive filetype and mimetype detection using libmagic and\ - \ Pygments." - homepage_url: "https://github.com/nexB/typecode" - binary_artifact: - url: "https://files.pythonhosted.org/packages/a1/3c/6d135ccbbf42230d3bef43bc052fd6993171a71ce1ec868c2581d8f930ab/typecode-30.0.2-py3-none-any.whl" - hash: - value: "06b1bffa93525acf4b6a52393fd0ee20916f62ddd40c0f2d27ca75a08231a46c" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d6/ca/c93584110e501e579e8901c9ad0c5db60dc69e84dfaf2224d5f95fca067b/typecode-30.0.2.tar.gz" - hash: - value: "17689d20af0ae6116e797ef2c5de65f0ce809128cf0e68479b34bd6ba4bc3898" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/typecode.git" - revision: "" - path: "" - - id: "PyPI::typecode-libmagic:5.39.210531" - purl: "pkg:pypi/typecode-libmagic@5.39.210531" - authors: - - "nexB " - declared_licenses: - - "apache-2.0 AND bsd-simplified-darwin AND (bsd-simplified AND public-domain\ - \ AND bsd-new AND isc AND (bsd-new OR gpl-1.0-plus) AND bsd-original)" - declared_licenses_processed: - unmapped: - - "apache-2.0 AND bsd-simplified-darwin AND (bsd-simplified AND public-domain\ - \ AND bsd-new AND isc AND (bsd-new OR gpl-1.0-plus) AND bsd-original)" - description: "A ScanCode path provider plugin to provide a prebuilt native libmagic\ - \ binary and database." - homepage_url: "https://github.com/nexB/scancode-plugins" - binary_artifact: - url: "https://files.pythonhosted.org/packages/89/bc/135d2c5a345f1c52431dbc92c181003ba61bcd35e304d36387e33070a9c5/typecode_libmagic-5.39.210531-py3-none-manylinux1_x86_64.whl" - hash: - value: "ee001c8093dfa89a9d1fe6d9139ef9f367a1cb9af6fd02ffebf9246af994fbf7" - algorithm: "SHA-256" - source_artifact: - url: "" - hash: - value: "" - algorithm: "" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/scancode-plugins.git" - revision: "" - path: "" - - id: "PyPI::uritools:4.0.3" - purl: "pkg:pypi/uritools@4.0.3" - authors: - - "Thomas Kemmer " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "URI parsing, classification and composition" - homepage_url: "https://github.com/tkem/uritools/" - binary_artifact: - url: "https://files.pythonhosted.org/packages/e6/17/5a4510d9ca9cc8be217ce359eb54e693dca81cf4d442308b282d5131b17d/uritools-4.0.3-py3-none-any.whl" - hash: - value: "bae297d090e69a0451130ffba6f2f1c9477244aa0a5543d66aed2d9f77d0dd9c" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/d3/43/4182fb2a03145e6d38698e38b49114ce59bc8c79063452eb585a58f8ce78/uritools-4.0.3.tar.gz" - hash: - value: "ee06a182a9c849464ce9d5fa917539aacc8edd2a4924d1b7aabeeecabcae3bc2" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/tkem/uritools.git" - revision: "" - path: "" - - id: "PyPI::urllib3:2.2.2" - purl: "pkg:pypi/urllib3@2.2.2" - authors: - - "Andrey Petrov " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "HTTP library with thread-safe connection pooling, file post, and\ - \ more." - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/ca/1c/89ffc63a9605b583d5df2be791a27bc1a42b7c32bab68d3c8f2f73a98cd4/urllib3-2.2.2-py3-none-any.whl" - hash: - value: "a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/43/6d/fa469ae21497ddc8bc93e5877702dca7cb8f911e337aca7452b5724f1bb6/urllib3-2.2.2.tar.gz" - hash: - value: "dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/urllib3/urllib3.git" - revision: "" - path: "" - - id: "PyPI::urlpy:0.5" - purl: "pkg:pypi/urlpy@0.5" - authors: - - "nexB Inc (based on code from Dan Lecocq)" - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Simple URL parsing, canonicalization and equivalence." - homepage_url: "http://github.com/nexB/urlpy" - binary_artifact: - url: "https://files.pythonhosted.org/packages/23/f0/43a8013e888f435c619f82b485ef8cf9fddfcceea7806d824b28d5ef8f76/urlpy-0.5-py2.py3-none-any.whl" - hash: - value: "841673d97e0dd7a4d7ba47abd49fa8e3a61709e189e40de1b04b150ce7c5ed9f" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/23/21/d176a137e4cc1ec4809534d116e9a06fc5fc0519077530ab5d040e356454/urlpy-0.5.tar.gz" - hash: - value: "e98ead47f4e422ca35080fd60a039f4546b7788bbba1b0a542a34c193dfba4bc" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/nexB/urlpy.git" - revision: "" - path: "" - - id: "PyPI::wcwidth:0.2.13" - purl: "pkg:pypi/wcwidth@0.2.13" - authors: - - "Jeff Quast " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Measures the displayed width of unicode strings in a terminal" - homepage_url: "https://github.com/jquast/wcwidth" - binary_artifact: - url: "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl" - hash: - value: "3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz" - hash: - value: "72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/jquast/wcwidth.git" - revision: "" - path: "" - - id: "PyPI::webencodings:0.5.1" - purl: "pkg:pypi/webencodings@0.5.1" - authors: - - "Geoffrey Sneddon " - declared_licenses: - - "BSD" - - "BSD License" - declared_licenses_processed: - spdx_expression: "BSD-3-Clause" - mapped: - BSD: "BSD-3-Clause" - BSD License: "BSD-3-Clause" - description: "Character encoding aliases for legacy web content" - homepage_url: "https://github.com/SimonSapin/python-webencodings" - binary_artifact: - url: "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl" - hash: - value: "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz" - hash: - value: "b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/SimonSapin/python-webencodings.git" - revision: "" - path: "" - - id: "PyPI::wheel:0.44.0" - purl: "pkg:pypi/wheel@0.44.0" - authors: - - "Daniel Holth " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "A built-package format for Python" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/1b/d1/9babe2ccaecff775992753d8686970b1e2755d21c8a63be73aba7a4e7d77/wheel-0.44.0-py3-none-any.whl" - hash: - value: "2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/b7/a0/95e9e962c5fd9da11c1e28aa4c0d8210ab277b1ada951d2aee336b505813/wheel-0.44.0.tar.gz" - hash: - value: "a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/pypa/wheel.git" - revision: "" - path: "" - - id: "PyPI::xmltodict:0.13.0" - purl: "pkg:pypi/xmltodict@0.13.0" - authors: - - "Martin Blech " - declared_licenses: - - "MIT" - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Makes working with XML feel like you are working with JSON" - homepage_url: "https://github.com/martinblech/xmltodict" - binary_artifact: - url: "https://files.pythonhosted.org/packages/94/db/fd0326e331726f07ff7f40675cd86aa804bfd2e5016c727fa761c934990e/xmltodict-0.13.0-py2.py3-none-any.whl" - hash: - value: "aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/39/0d/40df5be1e684bbaecdb9d1e0e40d5d482465de6b00cbb92b84ee5d243c7f/xmltodict-0.13.0.tar.gz" - hash: - value: "341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/martinblech/xmltodict.git" - revision: "" - path: "" - - id: "PyPI::zipp:3.20.0" - purl: "pkg:pypi/zipp@3.20.0" - authors: - - "\"Jason R. Coombs\" " - declared_licenses: - - "MIT License" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - MIT License: "MIT" - description: "Backport of pathlib-compatible object wrapper for zip files" - homepage_url: "" - binary_artifact: - url: "https://files.pythonhosted.org/packages/da/cc/b9958af9f9c86b51f846d8487440af495ecf19b16e426fce1ed0b0796175/zipp-3.20.0-py3-none-any.whl" - hash: - value: "58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d" - algorithm: "SHA-256" - source_artifact: - url: "https://files.pythonhosted.org/packages/0e/af/9f2de5bd32549a1b705af7a7c054af3878816a1267cb389c03cc4f342a51/zipp-3.20.0.tar.gz" - hash: - value: "0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31" - algorithm: "SHA-256" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "Git" - url: "https://github.com/jaraco/zipp.git" - revision: "" - path: "" - issues: - 'NPM::tests/data/package.json:': - - timestamp: "2024-08-17T19:07:21.849028339Z" - source: "NPM" - message: "NPM failed to resolve dependencies for path 'tests/data/package.json':\ - \ IllegalArgumentException: No lockfile found in 'tests/data'. This potentially\ - \ results in unstable versions of dependencies. To support this, enable\ - \ the 'allowDynamicVersions' option in 'config.yml'." - severity: "ERROR" - dependency_graphs: - NPM: - nodes: [] - edges: [] - PIP: - packages: - - "PyPI::alabaster:1.0.0" - - "PyPI::attrs:24.2.0" - - "PyPI::babel:2.16.0" - - "PyPI::backports-tarfile:1.2.0" - - "PyPI::banal:1.0.6" - - "PyPI::beartype:0.18.5" - - "PyPI::beautifulsoup4:4.12.3" - - "PyPI::binaryornot:0.4.4" - - "PyPI::boolean-py:4.0" - - "PyPI::build:1.2.1" - - "PyPI::cachetools:5.4.0" - - "PyPI::certifi:2024.7.4" - - "PyPI::cffi:1.17.0" - - "PyPI::chardet:5.2.0" - - "PyPI::charset-normalizer:3.3.2" - - "PyPI::click:8.1.7" - - "PyPI::colorama:0.4.6" - - "PyPI::commoncode:31.2.1" - - "PyPI::container-inspector:33.0.0" - - "PyPI::crc32c:2.6" - - "PyPI::cryptography:43.0.0" - - "PyPI::debian-inspector:31.1.0" - - "PyPI::dockerfile-parse:2.0.1" - - "PyPI::docutils:0.21.2" - - "PyPI::dparse2:0.7.0" - - "PyPI::dukpy:0.4.0" - - "PyPI::fasteners:0.19" - - "PyPI::filelock:3.15.4" - - "PyPI::fingerprints:1.2.3" - - "PyPI::ftfy:6.2.3" - - "PyPI::furo:2024.8.6" - - "PyPI::gemfileparser2:0.9.3" - - "PyPI::google-api-core:2.19.1" - - "PyPI::google-auth:2.33.0" - - "PyPI::googleapis-common-protos:1.63.2" - - "PyPI::grpcio-tools:1.65.5" - - "PyPI::grpcio:1.65.5" - - "PyPI::html5lib:1.1" - - "PyPI::idna:3.7" - - "PyPI::imagesize:1.4.1" - - "PyPI::importlib-metadata:8.2.0" - - "PyPI::importlib-resources:6.4.3" - - "PyPI::intbitset:3.1.0" - - "PyPI::isodate:0.6.1" - - "PyPI::jaraco-classes:3.4.0" - - "PyPI::jaraco-context:5.3.0" - - "PyPI::jaraco-functools:4.0.2" - - "PyPI::javaproperties:0.8.1" - - "PyPI::jeepney:0.8.0" - - "PyPI::jinja2:3.1.4" - - "PyPI::jsonstreams:0.6.0" - - "PyPI::keyring:25.3.0" - - "PyPI::license-expression:30.3.1" - - "PyPI::lxml:5.3.0" - - "PyPI::markdown-it-py:3.0.0" - - "PyPI::markupsafe:2.1.5" - - "PyPI::mdurl:0.1.2" - - "PyPI::more-itertools:10.4.0" - - "PyPI::mutf8:1.0.6" - - "PyPI::nh3:0.2.18" - - "PyPI::normality:2.5.0" - - "PyPI::packageurl-python:0.15.6" - - "PyPI::packaging:24.1" - - "PyPI::packvers:21.5" - - "PyPI::parameter-expansion-patched:0.3.1" - - "PyPI::pdfminer-six:20240706" - - "PyPI::pefile:2023.2.7" - - "PyPI::pip-requirements-parser:32.0.1" - - "PyPI::pkginfo2:30.0.0" - - "PyPI::pkginfo:1.10.0" - - "PyPI::pluggy:1.5.0" - - "PyPI::plugincode:32.0.0" - - "PyPI::ply:3.11" - - "PyPI::progress:1.6" - - "PyPI::proto-plus:1.24.0" - - "PyPI::protobuf:5.27.3" - - "PyPI::publicsuffix2:2.20191221" - - "PyPI::pyahocorasick:2.1.0" - - "PyPI::pyasn1-modules:0.4.0" - - "PyPI::pyasn1:0.6.0" - - "PyPI::pycparser:2.22" - - "PyPI::pygmars:0.8.1" - - "PyPI::pygments:2.18.0" - - "PyPI::pymaven-patch:0.3.2" - - "PyPI::pyopenssl:24.2.1" - - "PyPI::pypac:0.16.4" - - "PyPI::pyparsing:3.1.2" - - "PyPI::pyproject-hooks:1.1.0" - - "PyPI::pyyaml:6.0.2" - - "PyPI::rdflib:7.0.0" - - "PyPI::readme-renderer:44.0" - - "PyPI::requests-file:2.1.0" - - "PyPI::requests-toolbelt:1.0.0" - - "PyPI::requests:2.32.3" - - "PyPI::rfc3986:2.0.0" - - "PyPI::rich:13.7.1" - - "PyPI::rsa:4.9" - - "PyPI::saneyaml:0.6.1" - - "PyPI::scancode-toolkit-mini:32.2.1" - - "PyPI::secretstorage:3.3.3" - - "PyPI::semantic-version:2.10.0" - - "PyPI::setuptools:72.2.0" - - "PyPI::six:1.16.0" - - "PyPI::snowballstemmer:2.2.0" - - "PyPI::soupsieve:2.6" - - "PyPI::spdx-tools:0.8.2" - - "PyPI::sphinx-basic-ng:1.0.0b2" - - "PyPI::sphinx:8.0.2" - - "PyPI::sphinxcontrib-applehelp:2.0.0" - - "PyPI::sphinxcontrib-devhelp:2.0.0" - - "PyPI::sphinxcontrib-htmlhelp:2.1.0" - - "PyPI::sphinxcontrib-jsmath:1.0.1" - - "PyPI::sphinxcontrib-qthelp:2.0.0" - - "PyPI::sphinxcontrib-serializinghtml:2.0.0" - - "PyPI::text-unidecode:1.3" - - "PyPI::tldextract:5.1.2" - - "PyPI::toml:0.10.2" - - "PyPI::twine:5.1.1" - - "PyPI::typecode-libmagic:5.39.210531" - - "PyPI::typecode:30.0.2" - - "PyPI::uritools:4.0.3" - - "PyPI::urllib3:2.2.2" - - "PyPI::urlpy:0.5" - - "PyPI::wcwidth:0.2.13" - - "PyPI::webencodings:0.5.1" - - "PyPI::wheel:0.44.0" - - "PyPI::xmltodict:0.13.0" - - "PyPI::zipp:3.20.0" - scopes: - :data:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: - - root: 7 - - root: 19 - - root: 36 - - root: 73 - - root: 75 - - root: 93 - :docs-docs:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: - - root: 30 - :requirements-dev.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: - - root: 9 - - root: 35 - - root: 117 - - root: 125 - :requirements-scancode.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: - - root: 98 - - root: 118 - :requirements.txt:f5c0c3516ca0dc426f9c5411722e9632486ae362:install: - - root: 7 - - root: 19 - - root: 32 - - root: 36 - - root: 41 - - root: 73 - - root: 84 - - root: 85 - nodes: - - pkg: 13 - - pkg: 7 - - pkg: 19 - - pkg: 10 - - pkg: 79 - - pkg: 78 - - pkg: 96 - - pkg: 33 - - pkg: 75 - - pkg: 34 - - pkg: 74 - - pkg: 11 - - pkg: 14 - - pkg: 38 - - pkg: 121 - - pkg: 93 - - pkg: 32 - - pkg: 36 - - pkg: 41 - - pkg: 73 - - pkg: 80 - - pkg: 12 - - pkg: 20 - - pkg: 84 - - pkg: 58 - - pkg: 25 - - pkg: 27 - - pkg: 91 - - pkg: 115 - - pkg: 85 - - pkg: 1 - - pkg: 104 - - pkg: 6 - - pkg: 8 - - pkg: 15 - - pkg: 16 - - pkg: 88 - - pkg: 97 - - pkg: 114 - - pkg: 17 - - pkg: 22 - - pkg: 18 - - pkg: 21 - - pkg: 86 - - pkg: 63 - - pkg: 116 - - pkg: 24 - - pkg: 26 - - pkg: 4 - - pkg: 60 - - pkg: 28 - - pkg: 123 - - pkg: 29 - - pkg: 31 - - pkg: 102 - - pkg: 124 - - pkg: 37 - - pkg: 127 - - pkg: 40 - - pkg: 42 - - pkg: 57 - - pkg: 46 - - pkg: 47 - - pkg: 55 - - pkg: 49 - - pkg: 50 - - pkg: 52 - - pkg: 53 - - pkg: 61 - - pkg: 64 - - pkg: 65 - - pkg: 66 - - pkg: 62 - - pkg: 67 - - pkg: 68 - - pkg: 70 - - pkg: 71 - - pkg: 76 - - pkg: 77 - - pkg: 81 - - pkg: 82 - - pkg: 83 - - pkg: 5 - - pkg: 72 - - pkg: 43 - - pkg: 89 - - pkg: 100 - - pkg: 120 - - pkg: 126 - - pkg: 105 - - pkg: 119 - - pkg: 122 - - pkg: 98 - - pkg: 118 - - pkg: 87 - - pkg: 9 - - pkg: 101 - - pkg: 35 - - pkg: 44 - - pkg: 3 - - pkg: 45 - - pkg: 48 - - pkg: 99 - - pkg: 51 - - pkg: 69 - - pkg: 23 - - pkg: 59 - - pkg: 90 - - pkg: 92 - - pkg: 94 - - pkg: 56 - - pkg: 54 - - pkg: 95 - - pkg: 117 - - pkg: 125 - - {} - - pkg: 2 - - pkg: 39 - - pkg: 103 - - pkg: 108 - - pkg: 109 - - pkg: 110 - - pkg: 111 - - pkg: 112 - - pkg: 113 - - pkg: 107 - - pkg: 106 - - pkg: 30 - edges: - - from: 1 - to: 0 - - from: 5 - to: 4 - - from: 6 - to: 4 - - from: 7 - to: 3 - - from: 7 - to: 5 - - from: 7 - to: 6 - - from: 9 - to: 8 - - from: 10 - to: 8 - - from: 15 - to: 11 - - from: 15 - to: 12 - - from: 15 - to: 13 - - from: 15 - to: 14 - - from: 16 - to: 7 - - from: 16 - to: 8 - - from: 16 - to: 9 - - from: 16 - to: 10 - - from: 16 - to: 15 - - from: 21 - to: 20 - - from: 22 - to: 21 - - from: 23 - to: 22 - - from: 25 - to: 24 - - from: 27 - to: 15 - - from: 28 - to: 13 - - from: 28 - to: 15 - - from: 28 - to: 26 - - from: 28 - to: 27 - - from: 29 - to: 15 - - from: 29 - to: 25 - - from: 29 - to: 28 - - from: 32 - to: 31 - - from: 37 - to: 36 - - from: 39 - to: 15 - - from: 39 - to: 30 - - from: 39 - to: 32 - - from: 39 - to: 34 - - from: 39 - to: 37 - - from: 39 - to: 38 - - from: 41 - to: 30 - - from: 41 - to: 34 - - from: 41 - to: 39 - - from: 41 - to: 40 - - from: 42 - to: 0 - - from: 42 - to: 30 - - from: 44 - to: 43 - - from: 46 - to: 36 - - from: 46 - to: 44 - - from: 46 - to: 45 - - from: 49 - to: 0 - - from: 49 - to: 12 - - from: 49 - to: 38 - - from: 49 - to: 48 - - from: 50 - to: 49 - - from: 52 - to: 51 - - from: 56 - to: 54 - - from: 56 - to: 55 - - from: 58 - to: 57 - - from: 61 - to: 60 - - from: 64 - to: 63 - - from: 65 - to: 54 - - from: 66 - to: 33 - - from: 70 - to: 12 - - from: 70 - to: 22 - - from: 73 - to: 43 - - from: 73 - to: 72 - - from: 76 - to: 34 - - from: 76 - to: 39 - - from: 76 - to: 75 - - from: 81 - to: 15 - - from: 81 - to: 54 - - from: 81 - to: 67 - - from: 84 - to: 54 - - from: 85 - to: 43 - - from: 85 - to: 84 - - from: 89 - to: 34 - - from: 89 - to: 36 - - from: 89 - to: 66 - - from: 89 - to: 82 - - from: 89 - to: 83 - - from: 89 - to: 85 - - from: 89 - to: 86 - - from: 89 - to: 87 - - from: 89 - to: 88 - - from: 90 - to: 1 - - from: 90 - to: 30 - - from: 90 - to: 39 - - from: 90 - to: 70 - - from: 90 - to: 76 - - from: 91 - to: 77 - - from: 92 - to: 0 - - from: 92 - to: 15 - - from: 92 - to: 30 - - from: 92 - to: 32 - - from: 92 - to: 33 - - from: 92 - to: 34 - - from: 92 - to: 35 - - from: 92 - to: 37 - - from: 92 - to: 38 - - from: 92 - to: 39 - - from: 92 - to: 41 - - from: 92 - to: 42 - - from: 92 - to: 44 - - from: 92 - to: 45 - - from: 92 - to: 46 - - from: 92 - to: 47 - - from: 92 - to: 50 - - from: 92 - to: 52 - - from: 92 - to: 53 - - from: 92 - to: 56 - - from: 92 - to: 58 - - from: 92 - to: 59 - - from: 92 - to: 61 - - from: 92 - to: 62 - - from: 92 - to: 63 - - from: 92 - to: 64 - - from: 92 - to: 65 - - from: 92 - to: 66 - - from: 92 - to: 67 - - from: 92 - to: 68 - - from: 92 - to: 69 - - from: 92 - to: 70 - - from: 92 - to: 71 - - from: 92 - to: 73 - - from: 92 - to: 74 - - from: 92 - to: 75 - - from: 92 - to: 76 - - from: 92 - to: 77 - - from: 92 - to: 78 - - from: 92 - to: 79 - - from: 92 - to: 80 - - from: 92 - to: 81 - - from: 92 - to: 88 - - from: 92 - to: 89 - - from: 92 - to: 90 - - from: 92 - to: 91 - - from: 95 - to: 72 - - from: 95 - to: 94 - - from: 97 - to: 8 - - from: 97 - to: 17 - - from: 97 - to: 96 - - from: 98 - to: 60 - - from: 100 - to: 99 - - from: 102 - to: 22 - - from: 102 - to: 101 - - from: 103 - to: 58 - - from: 103 - to: 61 - - from: 103 - to: 98 - - from: 103 - to: 100 - - from: 103 - to: 101 - - from: 103 - to: 102 - - from: 107 - to: 80 - - from: 107 - to: 105 - - from: 107 - to: 106 - - from: 108 - to: 15 - - from: 111 - to: 110 - - from: 112 - to: 80 - - from: 112 - to: 111 - - from: 113 - to: 14 - - from: 113 - to: 15 - - from: 113 - to: 58 - - from: 113 - to: 103 - - from: 113 - to: 104 - - from: 113 - to: 107 - - from: 113 - to: 108 - - from: 113 - to: 109 - - from: 113 - to: 112 - - from: 125 - to: 15 - - from: 125 - to: 64 - - from: 125 - to: 72 - - from: 125 - to: 80 - - from: 125 - to: 105 - - from: 125 - to: 115 - - from: 125 - to: 116 - - from: 125 - to: 117 - - from: 125 - to: 118 - - from: 125 - to: 119 - - from: 125 - to: 120 - - from: 125 - to: 121 - - from: 125 - to: 122 - - from: 125 - to: 123 - - from: 125 - to: 124 - - from: 126 - to: 125 - - from: 127 - to: 32 - - from: 127 - to: 80 - - from: 127 - to: 125 - - from: 127 - to: 126 -scanner: null -advisor: null -evaluator: null -resolved_configuration: - package_curations: - - provider: - id: "DefaultDir" - curations: [] - - provider: - id: "DefaultFile" - curations: [] From a0731981ae923ff166a085b9a5f86fcd2e632e0a Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Thu, 29 Aug 2024 13:41:27 -0300 Subject: [PATCH 147/489] added links --- docs/source/index.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 320306f8..308f2ce9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -354,4 +354,9 @@ The Scanoss Open Source scanoss-py package is released under the MIT license. .. toctree:: :maxdepth: 2 - :caption: Contents: \ No newline at end of file + :hidden: + :caption: Links + + SCANOSS Website + GitHub + Software transparency foundation \ No newline at end of file From d9be5880d5ab0c5c31243b51fd96649b8043b533 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Thu, 29 Aug 2024 13:42:04 -0300 Subject: [PATCH 148/489] changed author --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ee17255c..9d25efb0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -8,7 +8,7 @@ project = 'Documentation for scanoss-py' copyright = '2024, Scan Open Source Solutions SL' -author = 'Jeronimo Ortiz' +author = 'Scan Open Source Solutions SL' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration From 2e8ce9e4503bfc23b4a8cdec8e7b577699892a5d Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Fri, 30 Aug 2024 13:45:32 -0300 Subject: [PATCH 149/489] updated file path for requirements --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a6d1391b..6d98edfe 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -29,4 +29,4 @@ sphinx: # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - - requirements: docs/source/requirements-docs.txt + - requirements: docs/requirements-docs.txt From f5273bd78f009485d0a32be84b6b813fb11de72e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 2 Sep 2024 16:30:19 +0200 Subject: [PATCH 150/489] feat: SP-1415 Initialized results command --- src/scanoss/cli.py | 69 +++++++++++++++++- src/scanoss/results.py | 130 ++++++++++++++++++++++++++++++++++ src/scanoss/utils/__init__.py | 0 src/scanoss/utils/colorize.py | 18 +++++ 4 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 src/scanoss/results.py create mode 100644 src/scanoss/utils/__init__.py create mode 100644 src/scanoss/utils/colorize.py diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 5e1783d3..9c6ee080 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -27,6 +27,7 @@ import pypac + from .scanner import Scanner from .scancodedeps import ScancodeDeps from .scantype import ScanType @@ -37,6 +38,7 @@ from .components import Components from . import __version__ from .scanner import FAST_WINNOWING +from .results import Results def print_stderr(*args, **kwargs): @@ -243,6 +245,36 @@ def setup_args() -> None: p_p_proxy.add_argument('--url', required=False, type=str, default="https://api.osskb.org", help='URL to test (default: https://api.osskb.org).') + p_results = subparsers.add_parser( + "results", + aliases=["res"], + description=f"SCANOSS Results commands: {__version__}", + help="Process scan results", + ) + p_results.add_argument( + "filepath", + metavar="FILEPATH", + type=str, + nargs="?", + help="Path to the file containing the results", + ) + p_results.add_argument( + "--match-type", + "-mt", + help="Filter results by match type (comma-separated, e.g., file,snippet,all)", + ) + p_results.add_argument( + "--status", + "-s", + help="Filter results by file status (comma-separated, e.g., pending, all)", + ) + p_results.add_argument( + "--prevent-commit", + action="store_true", + help="Prevents commit if there are any results available", + ) + p_results.set_defaults(func=results) + # Global Scan command options for p in [p_scan]: p.add_argument('--apiurl', type=str, @@ -288,7 +320,7 @@ def setup_args() -> None: # Help/Trace command options for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto, c_vulns, c_search, - c_versions, c_semgrep]: + c_versions, c_semgrep, p_results]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -878,6 +910,41 @@ def comp_versions(parser, args): exit(1) +def results(parser, args): + """ + Run the "results" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if not args.filepath: + print_stderr("Please specify a file that contains scan results") + parser.parse_args([args.subparser, "-h"]) + exit(1) + + results_file = f"{os.getcwd()}/{args.filepath}" + + if not os.path.isfile(results_file): + print_stderr(f"The specified file {args.filepath} does not exist") + exit(1) + + results = Results( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + file=results_file, + match_type=args.match_type, + status=args.status, + ).apply_filters() + + if args.prevent_commit: + results.check_for_precommit() + + + def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/results.py b/src/scanoss/results.py new file mode 100644 index 00000000..2d231352 --- /dev/null +++ b/src/scanoss/results.py @@ -0,0 +1,130 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2023, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json + +from scanoss.utils.colorize import colorize + +from .scanossbase import ScanossBase + +ARG_TO_FILTER_MAP = { + "match_type": "id", + "status": "status", +} + + +class Results(ScanossBase): + def __init__( + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + file: str = None, + match_type: str = None, + status: str = None, + ): + """ + Handles parsing of scan result file + + :param debug: Debug + :param trace: Trace + :param quiet: Quiet + :param filepath: Path to the results file + :param match_type: Comma separated list of match type filters + :param status: Comma separated list of status filters + """ + super().__init__(debug, trace, quiet) + self.data = self._load_and_transform(file) + self.filters = self._load_filters(match_type=match_type, status=status) + + def _load_file(self, file: str) -> dict: + with open(file, "r") as jsonfile: + try: + return json.load(jsonfile) + except Exception as e: + self.print_stderr(f"ERROR: Problem parsing input JSON: {e}") + + def _load_and_transform(self, file: str) -> list: + raw_data = self._load_file(file) + return self._transform_data(raw_data) + + def _transform_data(self, data: dict) -> list: + result = [] + for filename, file_data in data.items(): + if file_data: + file_obj = {"filename": filename} + file_obj.update(file_data[0]) + result.append(file_obj) + return result + + def _load_filters(self, **kwargs): + filters = {key: None for key in kwargs} + + for key, value in kwargs.items(): + if value: + filters[key] = self.__extract_comma_separated_values(value) + + return filters + + def __extract_comma_separated_values(self, values: str) -> dict: + return [value.strip() for value in values.split(",")] + + def apply_filters(self): + filtered_data = [] + for item in self.data: + if self._item_matches_filters(item): + filtered_data.append(item) + self.data = filtered_data + + return self + + def _item_matches_filters(self, item): + for filter_key, filter_value in self.filters.items(): + if not filter_value: + continue + + item_value = item.get(ARG_TO_FILTER_MAP[filter_key]) + if isinstance(filter_value, list): + if item_value not in filter_value: + return False + elif item_value != filter_value: + return False + return True + + def check_for_precommit(self): + if self.data: + self._present_precommit_overview() + exit(1) + return self + + def _present_precommit_overview(self): + self.print_stderr( + f"{colorize(f"ERROR: Found {len(self.data)} potential open source results that may need your attention:", "RED")}" + ) + for item in self.data: + self.print_stderr(f" - {item['filename']}") + + self.print_stderr( + f"Run {colorize('scanoss-lui', "GREEN")} in the terminal to view the results in more detail." + ) diff --git a/src/scanoss/utils/__init__.py b/src/scanoss/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/scanoss/utils/colorize.py b/src/scanoss/utils/colorize.py new file mode 100644 index 00000000..135bd571 --- /dev/null +++ b/src/scanoss/utils/colorize.py @@ -0,0 +1,18 @@ +COLORS = { + "RED": "\033[31m", + "GREEN": "\033[32m", + "YELLOW": "\033[33m", + "BLUE": "\033[34m", + "MAGENTA": "\033[35m", + "CYAN": "\033[36m", + "WHITE": "\033[37m", + "RESET": "\033[0m", +} + + +def colorize(text: str, color: str) -> str: + if color not in COLORS: + raise KeyError( + f"Invalid color '{color}'. Valid colors are: {', '.join(COLORS.keys())}" + ) + return f"{COLORS[color]}{text}{COLORS['RESET']}" From b5816be2fd76f4bafc2eb8452dce7b7223976646 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Sep 2024 08:27:55 +0200 Subject: [PATCH 151/489] feat: SP-1415 added enums for result filters, handle defaults, handle "all" filtering --- src/scanoss/cli.py | 2 +- src/scanoss/results.py | 61 ++++++++++++++++++++++++++++++++--- src/scanoss/utils/colorize.py | 28 ++++++++-------- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 9c6ee080..02b7553c 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -921,7 +921,7 @@ def results(parser, args): Parsed arguments """ if not args.filepath: - print_stderr("Please specify a file that contains scan results") + print_stderr('ERROR: Please specify a file containing the results') parser.parse_args([args.subparser, "-h"]) exit(1) diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 2d231352..be0c54dd 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -23,18 +23,44 @@ """ import json +from enum import Enum +from typing import Any, Dict from scanoss.utils.colorize import colorize from .scanossbase import ScanossBase + +class MatchType(Enum): + FILE = "file" + SNIPPET = "snippet" + ALL = "all" + + +class Status(Enum): + PENDING = "pending" + ALL = "all" + + +class FilterKey(Enum): + MATCH_TYPE = "match_type" + STATUS = "status" + + +AVAILABLE_FILTER_VALUES = { + FilterKey.MATCH_TYPE: [e.value for e in MatchType], + FilterKey.STATUS: [e.value for e in Status], +} + + ARG_TO_FILTER_MAP = { - "match_type": "id", - "status": "status", + FilterKey.MATCH_TYPE: "id", + FilterKey.STATUS: "status", } class Results(ScanossBase): + def __init__( self, debug: bool = False, @@ -58,7 +84,7 @@ def __init__( self.data = self._load_and_transform(file) self.filters = self._load_filters(match_type=match_type, status=status) - def _load_file(self, file: str) -> dict: + def _load_file(self, file: str) -> Dict[str, Any]: with open(file, "r") as jsonfile: try: return json.load(jsonfile) @@ -83,7 +109,10 @@ def _load_filters(self, **kwargs): for key, value in kwargs.items(): if value: - filters[key] = self.__extract_comma_separated_values(value) + if key.upper() in FilterKey.__members__: + filters[FilterKey[key.upper()]] = ( + self.__extract_comma_separated_values(value) + ) return filters @@ -104,25 +133,49 @@ def _item_matches_filters(self, item): if not filter_value: continue + self._validate_filter_values(filter_key, filter_value) + item_value = item.get(ARG_TO_FILTER_MAP[filter_key]) if isinstance(filter_value, list): + if filter_value == ["all"]: + continue if item_value not in filter_value: return False elif item_value != filter_value: return False return True + def _validate_filter_values(self, filter_key: FilterKey, filter_value: str): + if any( + value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) + for value in filter_value + ): + valid_values = ", ".join(AVAILABLE_FILTER_VALUES.get(filter_key, [])) + self.print_stderr( + f"ERROR: Invalid filter value '{filter_value}' for filter '{filter_key.value}'. " + f"Valid values are: {valid_values}" + ) + exit(1) + def check_for_precommit(self): + """ + Check for precommit and print results if data exists. + Raises an exception if potential open source results are found. + """ if self.data: self._present_precommit_overview() exit(1) + else: + self.print_stderr("No potential open source results found.") return self def _present_precommit_overview(self): self.print_stderr( f"{colorize(f"ERROR: Found {len(self.data)} potential open source results that may need your attention:", "RED")}" ) + for item in self.data: + self.print_stderr(f" - {item['filename']}") self.print_stderr( diff --git a/src/scanoss/utils/colorize.py b/src/scanoss/utils/colorize.py index 135bd571..9b7f368b 100644 --- a/src/scanoss/utils/colorize.py +++ b/src/scanoss/utils/colorize.py @@ -1,18 +1,20 @@ -COLORS = { - "RED": "\033[31m", - "GREEN": "\033[32m", - "YELLOW": "\033[33m", - "BLUE": "\033[34m", - "MAGENTA": "\033[35m", - "CYAN": "\033[36m", - "WHITE": "\033[37m", - "RESET": "\033[0m", -} +from enum import Enum + + +class Color(Enum): + RED = "\033[31m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + BLUE = "\033[34m" + MAGENTA = "\033[35m" + CYAN = "\033[36m" + WHITE = "\033[37m" + RESET = "\033[0m" def colorize(text: str, color: str) -> str: - if color not in COLORS: + if color not in Color.__members__: raise KeyError( - f"Invalid color '{color}'. Valid colors are: {', '.join(COLORS.keys())}" + f"Invalid color '{color}'. Valid colors are: {', '.join(Color._member_names_)}" ) - return f"{COLORS[color]}{text}{COLORS['RESET']}" + return f"{Color[color].value}{text}{Color.RESET.value}" From 149960d325876701f77ae2d05a14464fed61bda6 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Sep 2024 09:12:36 +0200 Subject: [PATCH 152/489] feat:SP-1415 handle default filters --- src/scanoss/cli.py | 2 +- src/scanoss/results.py | 29 +++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 02b7553c..bdd29587 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -930,7 +930,7 @@ def results(parser, args): if not os.path.isfile(results_file): print_stderr(f"The specified file {args.filepath} does not exist") exit(1) - + results = Results( debug=args.debug, trace=args.trace, diff --git a/src/scanoss/results.py b/src/scanoss/results.py index be0c54dd..2493dabf 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -24,7 +24,7 @@ import json from enum import Enum -from typing import Any, Dict +from typing import Any, Dict, List from scanoss.utils.colorize import colorize @@ -58,6 +58,11 @@ class FilterKey(Enum): FilterKey.STATUS: "status", } +DEFAULT_FILTERS = { + FilterKey.MATCH_TYPE: ["all"], + FilterKey.STATUS: ["pending"], +} + class Results(ScanossBase): @@ -80,6 +85,7 @@ def __init__( :param match_type: Comma separated list of match type filters :param status: Comma separated list of status filters """ + super().__init__(debug, trace, quiet) self.data = self._load_and_transform(file) self.filters = self._load_filters(match_type=match_type, status=status) @@ -91,7 +97,11 @@ def _load_file(self, file: str) -> Dict[str, Any]: except Exception as e: self.print_stderr(f"ERROR: Problem parsing input JSON: {e}") - def _load_and_transform(self, file: str) -> list: + def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: + """ + Load the file and transform the data into a list of dictionaries with the filename and the file data + """ + raw_data = self._load_file(file) return self._transform_data(raw_data) @@ -104,19 +114,26 @@ def _transform_data(self, data: dict) -> list: result.append(file_obj) return result - def _load_filters(self, **kwargs): + def _load_filters(self, **kwargs) -> Dict[FilterKey, List[str]]: filters = {key: None for key in kwargs} for key, value in kwargs.items(): if value: if key.upper() in FilterKey.__members__: filters[FilterKey[key.upper()]] = ( - self.__extract_comma_separated_values(value) + self._extract_comma_separated_values(value) ) + else: + if key == "match_type": + filters[FilterKey.MATCH_TYPE] = DEFAULT_FILTERS[ + FilterKey.MATCH_TYPE + ] + elif key == "status": + filters[FilterKey.STATUS] = DEFAULT_FILTERS[FilterKey.STATUS] return filters - def __extract_comma_separated_values(self, values: str) -> dict: + def _extract_comma_separated_values(self, values: str) -> dict: return [value.strip() for value in values.split(",")] def apply_filters(self): @@ -162,6 +179,7 @@ def check_for_precommit(self): Check for precommit and print results if data exists. Raises an exception if potential open source results are found. """ + if self.data: self._present_precommit_overview() exit(1) @@ -175,7 +193,6 @@ def _present_precommit_overview(self): ) for item in self.data: - self.print_stderr(f" - {item['filename']}") self.print_stderr( From 775b8a54907dee655775506c11978552d0ca763f Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Sep 2024 12:17:43 +0200 Subject: [PATCH 153/489] feat: SP-1415 add output format and file --- src/scanoss/cli.py | 25 ++++++-- src/scanoss/results.py | 127 ++++++++++++++++++++++------------------- 2 files changed, 87 insertions(+), 65 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index bdd29587..e4bee3fa 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -269,9 +269,20 @@ def setup_args() -> None: help="Filter results by file status (comma-separated, e.g., pending, all)", ) p_results.add_argument( - "--prevent-commit", + "--has-pending", action="store_true", - help="Prevents commit if there are any results available", + help="Filter results to only include files with pending status", + ) + p_results.add_argument( + "--output", + "-o", + help="Output result file", + ) + p_results.add_argument( + "--format", + "-f", + choices=["json", "plain"], + help="Output format", ) p_results.set_defaults(func=results) @@ -938,10 +949,14 @@ def results(parser, args): file=results_file, match_type=args.match_type, status=args.status, - ).apply_filters() + output_file=args.output, + output_format=args.format, + ) - if args.prevent_commit: - results.check_for_precommit() + if args.has_pending: + return results.get_pending_identifications().present() + + results.apply_filters().present() diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 2493dabf..7bfb24f8 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -23,46 +23,37 @@ """ import json -from enum import Enum from typing import Any, Dict, List from scanoss.utils.colorize import colorize from .scanossbase import ScanossBase - -class MatchType(Enum): - FILE = "file" - SNIPPET = "snippet" - ALL = "all" - - -class Status(Enum): - PENDING = "pending" - ALL = "all" - - -class FilterKey(Enum): - MATCH_TYPE = "match_type" - STATUS = "status" +MATCH_TYPES = ["file", "snippet"] +STATUSES = ["pending"] AVAILABLE_FILTER_VALUES = { - FilterKey.MATCH_TYPE: [e.value for e in MatchType], - FilterKey.STATUS: [e.value for e in Status], + "match_type": [e for e in MATCH_TYPES], + "status": [e for e in STATUSES], } ARG_TO_FILTER_MAP = { - FilterKey.MATCH_TYPE: "id", - FilterKey.STATUS: "status", + "match_type": "id", + "status": "status", } -DEFAULT_FILTERS = { - FilterKey.MATCH_TYPE: ["all"], - FilterKey.STATUS: ["pending"], +PENDING_IDENTIFICATION_FILTERS = { + "match_type": ["file", "snippet"], + "status": ["pending"], } +AVAILABLE_OUTPUT_FORMATS = ["json", "plain"] + +NO_RESULTS_MSG = "No potential open source results found." +FOUND_RESULTS_MSG = f"Run {colorize("scanoss-lui", "GREEN")} in the terminal to view the results in more detail." + class Results(ScanossBase): @@ -74,6 +65,8 @@ def __init__( file: str = None, match_type: str = None, status: str = None, + output_file: str = None, + output_format: str = None, ): """ Handles parsing of scan result file @@ -89,13 +82,15 @@ def __init__( super().__init__(debug, trace, quiet) self.data = self._load_and_transform(file) self.filters = self._load_filters(match_type=match_type, status=status) + self.output_file = output_file + self.output_format = output_format def _load_file(self, file: str) -> Dict[str, Any]: with open(file, "r") as jsonfile: try: return json.load(jsonfile) except Exception as e: - self.print_stderr(f"ERROR: Problem parsing input JSON: {e}") + print(f"ERROR: Problem parsing input JSON: {e}") def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: """ @@ -114,22 +109,12 @@ def _transform_data(self, data: dict) -> list: result.append(file_obj) return result - def _load_filters(self, **kwargs) -> Dict[FilterKey, List[str]]: + def _load_filters(self, **kwargs) -> Dict[str, List[str]]: filters = {key: None for key in kwargs} for key, value in kwargs.items(): if value: - if key.upper() in FilterKey.__members__: - filters[FilterKey[key.upper()]] = ( - self._extract_comma_separated_values(value) - ) - else: - if key == "match_type": - filters[FilterKey.MATCH_TYPE] = DEFAULT_FILTERS[ - FilterKey.MATCH_TYPE - ] - elif key == "status": - filters[FilterKey.STATUS] = DEFAULT_FILTERS[FilterKey.STATUS] + filters[key] = self._extract_comma_separated_values(value) return filters @@ -154,47 +139,69 @@ def _item_matches_filters(self, item): item_value = item.get(ARG_TO_FILTER_MAP[filter_key]) if isinstance(filter_value, list): - if filter_value == ["all"]: - continue if item_value not in filter_value: return False elif item_value != filter_value: return False return True - def _validate_filter_values(self, filter_key: FilterKey, filter_value: str): + def _validate_filter_values(self, filter_key: str, filter_value: str): if any( value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) for value in filter_value ): valid_values = ", ".join(AVAILABLE_FILTER_VALUES.get(filter_key, [])) - self.print_stderr( + raise Exception( f"ERROR: Invalid filter value '{filter_value}' for filter '{filter_key.value}'. " f"Valid values are: {valid_values}" ) - exit(1) - def check_for_precommit(self): - """ - Check for precommit and print results if data exists. - Raises an exception if potential open source results are found. - """ + def get_pending_identifications(self): + self.filters = PENDING_IDENTIFICATION_FILTERS + self.apply_filters() - if self.data: - self._present_precommit_overview() - exit(1) - else: - self.print_stderr("No potential open source results found.") return self - def _present_precommit_overview(self): - self.print_stderr( - f"{colorize(f"ERROR: Found {len(self.data)} potential open source results that may need your attention:", "RED")}" - ) + def present(self, output_format: str = None, output_file: str = None): + file_path = output_file or self.output_file + fmt = output_format or self.output_format - for item in self.data: - self.print_stderr(f" - {item['filename']}") + if fmt and fmt not in AVAILABLE_OUTPUT_FORMATS: + raise Exception( + f"ERROR: Invalid output format '{output_format}'. Valid values are: {', '.join(AVAILABLE_OUTPUT_FORMATS)}" + ) - self.print_stderr( - f"Run {colorize('scanoss-lui', "GREEN")} in the terminal to view the results in more detail." - ) + match fmt: + case "json": + return self._present_json(file_path) + case "plain": + return self._present_plain(file_path) + case _: + return self._present_stdout() + + def _present_json(self, file: str = None): + if not file: + return json.dumps(self.data, indent=4) + with open(file, "w") as f: + f.write(json.dumps(self.data, indent=4)) + + def _present_plain(self, file: str = None): + if not file: + return self._present_stdout() + with open(file, "w") as f: + for item in self.data: + f.write(f" - {item['filename']}\n") + if len(self.data): + f.write(FOUND_RESULTS_MSG) + else: + f.write(NO_RESULTS_MSG) + f.close() + + def _present_stdout(self): + if not self.data: + print(NO_RESULTS_MSG) + return + + for item in self.data: + print(f" - {item['filename']}") + print(FOUND_RESULTS_MSG) From 84b8fca7b1427875d0af52040f37b817ea0fc8db Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Sep 2024 12:44:10 +0200 Subject: [PATCH 154/489] feat: SP-1415 exit if has pending identifications --- src/scanoss/cli.py | 8 +++++--- src/scanoss/results.py | 20 ++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index e4bee3fa..9fa71955 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -261,12 +261,12 @@ def setup_args() -> None: p_results.add_argument( "--match-type", "-mt", - help="Filter results by match type (comma-separated, e.g., file,snippet,all)", + help="Filter results by match type (comma-separated, e.g., file,snippet)", ) p_results.add_argument( "--status", "-s", - help="Filter results by file status (comma-separated, e.g., pending, all)", + help="Filter results by file status (comma-separated, e.g., pending, identified)", ) p_results.add_argument( "--has-pending", @@ -954,7 +954,9 @@ def results(parser, args): ) if args.has_pending: - return results.get_pending_identifications().present() + results.get_pending_identifications().present() + if results.has_results(): + exit(1) results.apply_filters().present() diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 7bfb24f8..ca3a5d49 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -30,7 +30,7 @@ from .scanossbase import ScanossBase MATCH_TYPES = ["file", "snippet"] -STATUSES = ["pending"] +STATUSES = ["pending", "identified"] AVAILABLE_FILTER_VALUES = { @@ -51,9 +51,6 @@ AVAILABLE_OUTPUT_FORMATS = ["json", "plain"] -NO_RESULTS_MSG = "No potential open source results found." -FOUND_RESULTS_MSG = f"Run {colorize("scanoss-lui", "GREEN")} in the terminal to view the results in more detail." - class Results(ScanossBase): @@ -162,6 +159,9 @@ def get_pending_identifications(self): return self + def has_results(self): + return bool(self.data) + def present(self, output_format: str = None, output_file: str = None): file_path = output_file or self.output_file fmt = output_format or self.output_format @@ -181,9 +181,10 @@ def present(self, output_format: str = None, output_file: str = None): def _present_json(self, file: str = None): if not file: - return json.dumps(self.data, indent=4) + print(json.dumps(self.data, indent=2)) + return json.dumps(self.data, indent=2) with open(file, "w") as f: - f.write(json.dumps(self.data, indent=4)) + f.write(json.dumps(self.data, indent=2)) def _present_plain(self, file: str = None): if not file: @@ -191,17 +192,12 @@ def _present_plain(self, file: str = None): with open(file, "w") as f: for item in self.data: f.write(f" - {item['filename']}\n") - if len(self.data): - f.write(FOUND_RESULTS_MSG) - else: - f.write(NO_RESULTS_MSG) f.close() def _present_stdout(self): if not self.data: - print(NO_RESULTS_MSG) + print("No potential open source results found.") return for item in self.data: print(f" - {item['filename']}") - print(FOUND_RESULTS_MSG) From f82a0e1c9c7e559d50b99e55eb8dc0e4ce513b75 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Sep 2024 12:46:48 +0200 Subject: [PATCH 155/489] feat: SP-1415 remove unused colorize util --- src/scanoss/results.py | 2 -- src/scanoss/utils/__init__.py | 0 src/scanoss/utils/colorize.py | 20 -------------------- 3 files changed, 22 deletions(-) delete mode 100644 src/scanoss/utils/__init__.py delete mode 100644 src/scanoss/utils/colorize.py diff --git a/src/scanoss/results.py b/src/scanoss/results.py index ca3a5d49..f713a824 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -25,8 +25,6 @@ import json from typing import Any, Dict, List -from scanoss.utils.colorize import colorize - from .scanossbase import ScanossBase MATCH_TYPES = ["file", "snippet"] diff --git a/src/scanoss/utils/__init__.py b/src/scanoss/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/scanoss/utils/colorize.py b/src/scanoss/utils/colorize.py deleted file mode 100644 index 9b7f368b..00000000 --- a/src/scanoss/utils/colorize.py +++ /dev/null @@ -1,20 +0,0 @@ -from enum import Enum - - -class Color(Enum): - RED = "\033[31m" - GREEN = "\033[32m" - YELLOW = "\033[33m" - BLUE = "\033[34m" - MAGENTA = "\033[35m" - CYAN = "\033[36m" - WHITE = "\033[37m" - RESET = "\033[0m" - - -def colorize(text: str, color: str) -> str: - if color not in Color.__members__: - raise KeyError( - f"Invalid color '{color}'. Valid colors are: {', '.join(Color._member_names_)}" - ) - return f"{Color[color].value}{text}{Color.RESET.value}" From 02ba54d048365f0a3448b8bae8171308a11d1a77 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Sep 2024 13:56:35 +0200 Subject: [PATCH 156/489] feat: SP-1415 modified version, updated changelog --- CHANGELOG.md | 8 ++++++++ src/scanoss/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1ce2c91..27c19d80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.15.0] - 2024-09-03 +### Added +- Added Results sub-command: + - Get all results (`scanoss-py results /path/to/file`) + - Get filtered results (`scanoss-py results /path/to/file --match-type=file,snippet status=pending`) + - Get pending declarations (`scanoss-py results /path/to/file --has-pending`) + ## [1.14.0] - 2024-08-09 ### Added - Added support for Python3.12 @@ -338,3 +345,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.12.3]: https://github.com/scanoss/scanoss.py/compare/v1.12.2...v1.12.3 [1.13.0]: https://github.com/scanoss/scanoss.py/compare/v1.12.3...v1.13.0 [1.14.0]: https://github.com/scanoss/scanoss.py/compare/v1.13.0...v1.14.0 +[1.15.0]: https://github.com/scanoss/scanoss.py/compare/v1.14.0...v1.15.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 217cabc6..d10a705a 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.14.0' +__version__ = "1.15.0" From 145a95505111977c17ada2ef4622771000aa1ccf Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Sep 2024 16:49:06 +0200 Subject: [PATCH 157/489] feat: SP-1415 remove match statement --- src/scanoss/results.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/scanoss/results.py b/src/scanoss/results.py index f713a824..c5bef1db 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -169,13 +169,12 @@ def present(self, output_format: str = None, output_file: str = None): f"ERROR: Invalid output format '{output_format}'. Valid values are: {', '.join(AVAILABLE_OUTPUT_FORMATS)}" ) - match fmt: - case "json": - return self._present_json(file_path) - case "plain": - return self._present_plain(file_path) - case _: - return self._present_stdout() + if fmt == "json": + return self._present_json(file_path) + elif fmt == "plain": + return self._present_plain(file_path) + else: + return self._present_stdout() def _present_json(self, file: str = None): if not file: From a19d3a6293db04e6fdee4d0cd2566a35c47165ab Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 4 Sep 2024 12:18:39 +0200 Subject: [PATCH 158/489] feat: SP-1415 format output --- src/scanoss/cli.py | 8 +++----- src/scanoss/results.py | 46 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 9fa71955..7396c604 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -941,7 +941,7 @@ def results(parser, args): if not os.path.isfile(results_file): print_stderr(f"The specified file {args.filepath} does not exist") exit(1) - + results = Results( debug=args.debug, trace=args.trace, @@ -955,11 +955,9 @@ def results(parser, args): if args.has_pending: results.get_pending_identifications().present() - if results.has_results(): - exit(1) - - results.apply_filters().present() + return results.has_results() + return results.apply_filters().present() def main(): diff --git a/src/scanoss/results.py b/src/scanoss/results.py index c5bef1db..c738b409 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -85,7 +85,7 @@ def _load_file(self, file: str) -> Dict[str, Any]: try: return json.load(jsonfile) except Exception as e: - print(f"ERROR: Problem parsing input JSON: {e}") + self.print_stderr(f"ERROR: Problem parsing input JSON: {e}") def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: """ @@ -177,24 +177,58 @@ def present(self, output_format: str = None, output_file: str = None): return self._present_stdout() def _present_json(self, file: str = None): + formatted_data = self._format_json_output() + output = json.dumps(formatted_data, indent=2) + if not file: - print(json.dumps(self.data, indent=2)) - return json.dumps(self.data, indent=2) + self.print_msg(output) + return output with open(file, "w") as f: - f.write(json.dumps(self.data, indent=2)) + f.write(output) def _present_plain(self, file: str = None): if not file: return self._present_stdout() with open(file, "w") as f: + f.write( + f"======== Found {len(self.data)} potential open source results ========\n" + ) for item in self.data: - f.write(f" - {item['filename']}\n") + f.write("\n") + f.write(self._format_plain_output_item(item)) f.close() def _present_stdout(self): if not self.data: print("No potential open source results found.") return + self.print_msg( + f"======== Found {len(self.data)} potential open source results ========\n" + ) + for item in self.data: + print(self._format_plain_output_item(item)) + def _format_json_output(self): + """ + Format the output data into a JSON object + """ + + formatted_data = [] for item in self.data: - print(f" - {item['filename']}") + formatted_data.append( + { + "file": item["filename"], + "status": item["status"] if "status" in item else None, + "match_type": item["id"], + "matched": item["matched"] if "matched" in item else None, + } + ) + return {"results": formatted_data, "total": len(formatted_data)} + + def _format_plain_output_item(self, item): + return ( + f"File: {item['filename']}\n" + f"Match type: {item['id']}\n" + f"Status: {item.get('status', 'N/A')}\n" + f"Matched: {item.get('matched', 'N/A')}\n" + ) From 8d4cdd306e8cd898b99f1735fcb3a26693996c89 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 4 Sep 2024 12:40:16 +0200 Subject: [PATCH 159/489] feat: SP-1415 return error if found pending results --- src/scanoss/cli.py | 3 ++- src/scanoss/results.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 7396c604..e5ba323c 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -955,7 +955,8 @@ def results(parser, args): if args.has_pending: results.get_pending_identifications().present() - return results.has_results() + if results.has_results(): + exit(1) return results.apply_filters().present() diff --git a/src/scanoss/results.py b/src/scanoss/results.py index c738b409..54573f97 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -151,7 +151,7 @@ def _validate_filter_values(self, filter_key: str, filter_value: str): f"Valid values are: {valid_values}" ) - def get_pending_identifications(self): + def get_pending_identifications(self) -> bool: self.filters = PENDING_IDENTIFICATION_FILTERS self.apply_filters() @@ -200,13 +200,13 @@ def _present_plain(self, file: str = None): def _present_stdout(self): if not self.data: - print("No potential open source results found.") + self.print_msg("No potential open source results found.") return self.print_msg( f"======== Found {len(self.data)} potential open source results ========\n" ) for item in self.data: - print(self._format_plain_output_item(item)) + self.print_msg(self._format_plain_output_item(item)) def _format_json_output(self): """ From da65b7549ae866102e3e88ec27d3137d4e9f7572 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 9 Sep 2024 13:01:30 +0200 Subject: [PATCH 160/489] feat: SP-1415 replace double quotes, print results to stdout by default --- src/scanoss/cli.py | 48 ++++++++++++------------ src/scanoss/results.py | 77 ++++++++++++++++++-------------------- src/scanoss/scanossbase.py | 21 +++++++++++ 3 files changed, 81 insertions(+), 65 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index e5ba323c..d48d1ea1 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -246,43 +246,43 @@ def setup_args() -> None: help='URL to test (default: https://api.osskb.org).') p_results = subparsers.add_parser( - "results", - aliases=["res"], + 'results', + aliases=['res'], description=f"SCANOSS Results commands: {__version__}", - help="Process scan results", + help='Process scan results', ) p_results.add_argument( - "filepath", - metavar="FILEPATH", + 'filepath', + metavar='FILEPATH', type=str, - nargs="?", - help="Path to the file containing the results", + nargs='?', + help='Path to the file containing the results', ) p_results.add_argument( - "--match-type", - "-mt", - help="Filter results by match type (comma-separated, e.g., file,snippet)", + '--match-type', + '-mt', + help='Filter results by match type (comma-separated, e.g., file,snippet)', ) p_results.add_argument( - "--status", - "-s", - help="Filter results by file status (comma-separated, e.g., pending, identified)", + '--status', + '-s', + help='Filter results by file status (comma-separated, e.g., pending, identified)', ) p_results.add_argument( - "--has-pending", - action="store_true", - help="Filter results to only include files with pending status", + '--has-pending', + action='store_true', + help='Filter results to only include files with pending status', ) p_results.add_argument( - "--output", - "-o", - help="Output result file", + '--output', + '-o', + help='Output result file', ) p_results.add_argument( - "--format", - "-f", - choices=["json", "plain"], - help="Output format", + '--format', + '-f', + choices=['json', 'plain'], + help='Output format (default: plain)', ) p_results.set_defaults(func=results) @@ -946,7 +946,7 @@ def results(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - file=results_file, + filepath=results_file, match_type=args.match_type, status=args.status, output_file=args.output, diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 54573f97..4af43aeb 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -57,7 +57,7 @@ def __init__( debug: bool = False, trace: bool = False, quiet: bool = False, - file: str = None, + filepath: str = None, match_type: str = None, status: str = None, output_file: str = None, @@ -75,7 +75,7 @@ def __init__( """ super().__init__(debug, trace, quiet) - self.data = self._load_and_transform(file) + self.data = self._load_and_transform(filepath) self.filters = self._load_filters(match_type=match_type, status=status) self.output_file = output_file self.output_format = output_format @@ -95,11 +95,12 @@ def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: raw_data = self._load_file(file) return self._transform_data(raw_data) - def _transform_data(self, data: dict) -> list: + @staticmethod + def _transform_data(data: dict) -> list: result = [] for filename, file_data in data.items(): if file_data: - file_obj = {"filename": filename} + file_obj = {'filename': filename} file_obj.update(file_data[0]) result.append(file_obj) return result @@ -113,7 +114,8 @@ def _load_filters(self, **kwargs) -> Dict[str, List[str]]: return filters - def _extract_comma_separated_values(self, values: str) -> dict: + @staticmethod + def _extract_comma_separated_values(values: str) -> dict: return [value.strip() for value in values.split(",")] def apply_filters(self): @@ -169,44 +171,17 @@ def present(self, output_format: str = None, output_file: str = None): f"ERROR: Invalid output format '{output_format}'. Valid values are: {', '.join(AVAILABLE_OUTPUT_FORMATS)}" ) - if fmt == "json": + if fmt == 'json': return self._present_json(file_path) - elif fmt == "plain": + elif fmt == 'plain': return self._present_plain(file_path) else: return self._present_stdout() def _present_json(self, file: str = None): - formatted_data = self._format_json_output() - output = json.dumps(formatted_data, indent=2) - - if not file: - self.print_msg(output) - return output - with open(file, "w") as f: - f.write(output) - - def _present_plain(self, file: str = None): - if not file: - return self._present_stdout() - with open(file, "w") as f: - f.write( - f"======== Found {len(self.data)} potential open source results ========\n" - ) - for item in self.data: - f.write("\n") - f.write(self._format_plain_output_item(item)) - f.close() - - def _present_stdout(self): - if not self.data: - self.print_msg("No potential open source results found.") - return - self.print_msg( - f"======== Found {len(self.data)} potential open source results ========\n" + self.print_to_file_or_stdout( + json.dumps(self._format_json_output(), indent=2), file ) - for item in self.data: - self.print_msg(self._format_plain_output_item(item)) def _format_json_output(self): """ @@ -217,13 +192,33 @@ def _format_json_output(self): for item in self.data: formatted_data.append( { - "file": item["filename"], - "status": item["status"] if "status" in item else None, - "match_type": item["id"], - "matched": item["matched"] if "matched" in item else None, + 'file': item['filename'], + 'status': item['status'] if 'status' in item else None, + 'match_type': item['id'], + 'matched': item['matched'] if 'matched' in item else None, } ) - return {"results": formatted_data, "total": len(formatted_data)} + return {'results': formatted_data, 'total': len(formatted_data)} + + def _present_plain(self, file: str = None): + if not self.data: + return self.print_stderr("No results to present") + self.print_to_file_or_stdout(self._format_plain_output(), file) + + def _present_stdout(self): + if not self.data: + return self.print_stderr("No results to present") + self.print_to_file_or_stdout(self._format_plain_output()) + + def _format_plain_output(self): + """ + Format the output data into a plain text + """ + + formatted = "" + for item in self.data: + formatted += f"{self._format_plain_output_item(item)} \n" + return formatted def _format_plain_output_item(self, item): return ( diff --git a/src/scanoss/scanossbase.py b/src/scanoss/scanossbase.py index b4941617..1694bfe3 100644 --- a/src/scanoss/scanossbase.py +++ b/src/scanoss/scanossbase.py @@ -68,3 +68,24 @@ def print_trace(self, *args, **kwargs): """ if self.trace: self.print_stderr(*args, **kwargs) + + @staticmethod + def print_stdout(*args, **kwargs): + """ + Print message to STDOUT + """ + print( + *args, + file=sys.stdout, + **kwargs, + ) + + def print_to_file_or_stdout(self, msg: str, file: str = None): + """ + Print message to file if provided or stdout + """ + if file: + with open(file, "w") as f: + f.write(msg) + else: + self.print_stdout(msg) From cd064ecf334f890eac1914d20a4250340613133f Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 10 Sep 2024 15:46:26 +0200 Subject: [PATCH 161/489] feat: SP-1415 small fixes --- src/scanoss/cli.py | 4 ++-- src/scanoss/results.py | 26 ++++++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index d48d1ea1..ab0f2afa 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -957,8 +957,8 @@ def results(parser, args): results.get_pending_identifications().present() if results.has_results(): exit(1) - - return results.apply_filters().present() + else: + results.apply_filters().present() def main(): diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 4af43aeb..193b85cf 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -105,8 +105,8 @@ def _transform_data(data: dict) -> list: result.append(file_obj) return result - def _load_filters(self, **kwargs) -> Dict[str, List[str]]: - filters = {key: None for key in kwargs} + def _load_filters(self, **kwargs): + filters = {} for key, value in kwargs.items(): if value: @@ -115,7 +115,7 @@ def _load_filters(self, **kwargs) -> Dict[str, List[str]]: return filters @staticmethod - def _extract_comma_separated_values(values: str) -> dict: + def _extract_comma_separated_values(values: str): return [value.strip() for value in values.split(",")] def apply_filters(self): @@ -128,21 +128,22 @@ def apply_filters(self): return self def _item_matches_filters(self, item): - for filter_key, filter_value in self.filters.items(): - if not filter_value: + for filter_key, filter_values in self.filters.items(): + if not filter_values: continue - self._validate_filter_values(filter_key, filter_value) + self._validate_filter_values(filter_key, filter_values) item_value = item.get(ARG_TO_FILTER_MAP[filter_key]) - if isinstance(filter_value, list): - if item_value not in filter_value: + if isinstance(filter_values, list): + if item_value not in filter_values: return False - elif item_value != filter_value: + elif item_value != filter_values: return False return True - def _validate_filter_values(self, filter_key: str, filter_value: str): + @staticmethod + def _validate_filter_values(filter_key: str, filter_value: list[str]): if any( value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) for value in filter_value @@ -153,7 +154,7 @@ def _validate_filter_values(self, filter_key: str, filter_value: str): f"Valid values are: {valid_values}" ) - def get_pending_identifications(self) -> bool: + def get_pending_identifications(self): self.filters = PENDING_IDENTIFICATION_FILTERS self.apply_filters() @@ -220,7 +221,8 @@ def _format_plain_output(self): formatted += f"{self._format_plain_output_item(item)} \n" return formatted - def _format_plain_output_item(self, item): + @staticmethod + def _format_plain_output_item(item): return ( f"File: {item['filename']}\n" f"Match type: {item['id']}\n" From a51c5567b7bc3d9b809a1fcd33c0bd9c39967636 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 12 Sep 2024 12:59:42 +0200 Subject: [PATCH 162/489] feat: SP-1415 add purl and license to results command output --- src/scanoss/results.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 193b85cf..29e1875c 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -143,7 +143,7 @@ def _item_matches_filters(self, item): return True @staticmethod - def _validate_filter_values(filter_key: str, filter_value: list[str]): + def _validate_filter_values(filter_key: str, filter_value: List[str]): if any( value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) for value in filter_value @@ -193,10 +193,16 @@ def _format_json_output(self): for item in self.data: formatted_data.append( { - 'file': item['filename'], - 'status': item['status'] if 'status' in item else None, + 'file': item.get('filename'), + 'status': item.get('status', "N/A"), 'match_type': item['id'], - 'matched': item['matched'] if 'matched' in item else None, + 'matched': item.get('matched', "N/A"), + 'purl': (item.get('purl')[0] if item.get('purl') else "N/A"), + 'license': ( + item.get('licenses')[0].get('name', "N/A") + if item.get('licenses') + else "N/A" + ), } ) return {'results': formatted_data, 'total': len(formatted_data)} @@ -223,9 +229,14 @@ def _format_plain_output(self): @staticmethod def _format_plain_output_item(item): + purls = item.get('purl', []) + licenses = item.get('licenses', []) + return ( - f"File: {item['filename']}\n" - f"Match type: {item['id']}\n" + f"File: {item.get('filename')}\n" + f"Match type: {item.get('id')}\n" f"Status: {item.get('status', 'N/A')}\n" f"Matched: {item.get('matched', 'N/A')}\n" + f"Purl: {purls[0] if purls else 'N/A'}\n" + f"License: {licenses[0].get('name', 'N/A') if licenses else 'N/A'}\n" ) From af1fb71ca89a022f46714d784a9b24662de784eb Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 12 Sep 2024 14:45:51 +0200 Subject: [PATCH 163/489] feat: SP-1415 added client help and class documentation --- CLIENT_HELP.md | 22 ++++++++++++ src/scanoss/results.py | 79 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 91 insertions(+), 10 deletions(-) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 3adf283b..1f5301c0 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -276,3 +276,25 @@ scanoss-py comp semgrep --key $SC_API_KEY -i purl-input.json -o semgrep-issues.j ``` **Note:** This sub-command requires a subscription to SCANOSS premium data. +### Results Commands +The `results` command provides the capability to operate on scan results. For example: + +The following command gets the pending results from a scan: +```bash +scanoss-py results results.json --has-pending +``` + +You can indicate the output format and an output file: +```bash +scanoss-py results results.json --format json --output results-output.json +``` + +You can also filter the results by either status or match type: +```bash +scanoss-py results results.json --status pending --match-type file +``` + +You can provide a comma separated list of statuses or match types: +```bash +scanoss-py results results.json --status pending,identified --match-type file,snippet +``` \ No newline at end of file diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 29e1875c..dc790149 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -51,6 +51,10 @@ class Results(ScanossBase): + """ + SCANOSS Results class \n + Handles the parsing and filtering of the scan results + """ def __init__( self, @@ -63,15 +67,17 @@ def __init__( output_file: str = None, output_format: str = None, ): - """ - Handles parsing of scan result file - - :param debug: Debug - :param trace: Trace - :param quiet: Quiet - :param filepath: Path to the results file - :param match_type: Comma separated list of match type filters - :param status: Comma separated list of status filters + """Initialise the Results class + + Args: + debug (bool, optional): Debug. Defaults to False. + trace (bool, optional): Trace. Defaults to False. + quiet (bool, optional): Quiet. Defaults to False. + filepath (str, optional): Path to the scan results file. Defaults to None. + match_type (str, optional): Comma separated match type filters. Defaults to None. + status (str, optional): Comma separated status filters. Defaults to None. + output_file (str, optional): Path to the output file. Defaults to None. + output_format (str, optional): Output format. Defaults to None. """ super().__init__(debug, trace, quiet) @@ -81,6 +87,14 @@ def __init__( self.output_format = output_format def _load_file(self, file: str) -> Dict[str, Any]: + """Load the JSON file + + Args: + file (str): Path to the JSON file + + Returns: + Dict[str, Any]: The parsed JSON data + """ with open(file, "r") as jsonfile: try: return json.load(jsonfile) @@ -97,6 +111,14 @@ def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: @staticmethod def _transform_data(data: dict) -> list: + """Transform the data into a list of dictionaries with the filename and the file data + + Args: + data (dict): The raw data + + Returns: + list: The transformed data + """ result = [] for filename, file_data in data.items(): if file_data: @@ -106,6 +128,11 @@ def _transform_data(data: dict) -> list: return result def _load_filters(self, **kwargs): + """Extract and parse the filters + + Returns: + dict: Parsed filters + """ filters = {} for key, value in kwargs.items(): @@ -119,6 +146,7 @@ def _extract_comma_separated_values(values: str): return [value.strip() for value in values.split(",")] def apply_filters(self): + """Apply the filters to the data""" filtered_data = [] for item in self.data: if self._item_matches_filters(item): @@ -155,6 +183,7 @@ def _validate_filter_values(filter_key: str, filter_value: List[str]): ) def get_pending_identifications(self): + """Get files with 'pending' status and 'file' or 'snippet' match type""" self.filters = PENDING_IDENTIFICATION_FILTERS self.apply_filters() @@ -164,6 +193,18 @@ def has_results(self): return bool(self.data) def present(self, output_format: str = None, output_file: str = None): + """Format and present the results. If no output format is provided, the results will be printed to stdout + + Args: + output_format (str, optional): Output format. Defaults to None. + output_file (str, optional): Output file. Defaults to None. + + Raises: + Exception: Invalid output format + + Returns: + None + """ file_path = output_file or self.output_file fmt = output_format or self.output_format @@ -180,6 +221,11 @@ def present(self, output_format: str = None, output_file: str = None): return self._present_stdout() def _present_json(self, file: str = None): + """Present the results in JSON format + + Args: + file (str, optional): Output file. Defaults to None. + """ self.print_to_file_or_stdout( json.dumps(self._format_json_output(), indent=2), file ) @@ -208,18 +254,31 @@ def _format_json_output(self): return {'results': formatted_data, 'total': len(formatted_data)} def _present_plain(self, file: str = None): + """Present the results in plain text format + + Args: + file (str, optional): Output file. Defaults to None. + + Returns: + None + """ if not self.data: return self.print_stderr("No results to present") self.print_to_file_or_stdout(self._format_plain_output(), file) def _present_stdout(self): + """Present the results to stdout + + Returns: + None + """ if not self.data: return self.print_stderr("No results to present") self.print_to_file_or_stdout(self._format_plain_output()) def _format_plain_output(self): """ - Format the output data into a plain text + Format the output data into a plain text string """ formatted = "" From 8bc987caffbfc603af1643fa99ad08cbf10e7c81 Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Thu, 12 Sep 2024 19:13:58 +0200 Subject: [PATCH 164/489] feat: SP-1448 Add support for scanoss.json settings file (#51) * feat: SP-1448 initialized ScanSettings class * feat: SP-1448 support legacy and new scan settings files * feat: SP-1448 working scanoss api with new scan settings files * feat: SP-1448 add missing imports * feat: SP-1448 fix tests * feat: SP-1448 updated copyright year, removed not used field param from docs * feat: SP-1448 updated copyright year, removed not used field param from docs * feat: SP-1489 initialize scan post processor class * feat: SP-1489 handle error if failed to parse raw json results * feat: SP-1489 add test for scan post processor * feat: SP-1489 add docs, fix typos --- src/scanoss/cli.py | 179 ++++++++++++++++++++-------- src/scanoss/results.py | 2 +- src/scanoss/scanner.py | 52 ++++---- src/scanoss/scanoss_settings.py | 198 +++++++++++++++++++++++++++++++ src/scanoss/scanossapi.py | 30 ++--- src/scanoss/scanpostprocessor.py | 104 ++++++++++++++++ tests/scanpostprocessor-test.py | 60 ++++++++++ 7 files changed, 532 insertions(+), 93 deletions(-) create mode 100644 src/scanoss/scanoss_settings.py create mode 100644 src/scanoss/scanpostprocessor.py create mode 100644 tests/scanpostprocessor-test.py diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ab0f2afa..08ddd6b3 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -29,7 +29,9 @@ from .scanner import Scanner +from .scanoss_settings import ScanossSettings from .scancodedeps import ScancodeDeps +from .scanner import FAST_WINNOWING, Scanner from .scantype import ScanType from .filecount import FileCount from .cyclonedx import CycloneDx @@ -102,6 +104,11 @@ def setup_args() -> None: help='Scancode command and path if required (optional - default scancode).') p_scan.add_argument('--sc-timeout', type=int, default=600, help='Timeout (in seconds) for scancode to complete (optional - default 600)') + p_scan.add_argument( + '--settings', + type=str, + help='Settings file to use for scanning (optional - default scanoss.json)', + ) # Sub-command: fingerprint p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'], @@ -489,42 +496,70 @@ def scan(parser, args): args: Namespace Parsed arguments """ - if not args.scan_dir and not args.wfp and not args.stdin and not args.dep and not args.files: - print_stderr('Please specify a file/folder, files (--files), fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)') + if ( + not args.scan_dir + and not args.wfp + and not args.stdin + and not args.dep + and not args.files + ): + print_stderr( + 'Please specify a file/folder, files (--files), fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)' + ) parser.parse_args([args.subparser, '-h']) exit(1) if args.pac and args.proxy: print_stderr('Please specify one of --proxy or --pac, not both') parser.parse_args([args.subparser, '-h']) exit(1) - scan_type: str = None - sbom_path: str = None + + if args.identify and args.settings: + print_stderr(f'ERROR: Cannot specify both --identify and --settings options.') + exit(1) + + def is_valid_file(file_path: str) -> bool: + if not os.path.exists(file_path) or not os.path.isfile(file_path): + print_stderr(f'Specified file does not exist or is not a file: {file_path}') + return False + if not Scanner.valid_json_file(file_path): + return False + return True + + scan_settings = ScanossSettings( + debug=args.debug, trace=args.trace, quiet=args.quiet + ) + if args.identify: - sbom_path = args.identify - scan_type = 'identify' - if not os.path.exists(sbom_path) or not os.path.isfile(sbom_path): - print_stderr(f'Specified --identify file does not exist or is not a file: {sbom_path}') - exit(1) - if not Scanner.valid_json_file(sbom_path): # Make sure it's a valid JSON file + if not is_valid_file(args.identify) or args.ignore: exit(1) - if args.ignore: - print_stderr(f'Warning: Specified --identify and --ignore options. Skipping ignore.') + scan_settings.load_json_file(args.identify).set_file_type( + 'legacy' + ).set_scan_type('identify') elif args.ignore: - sbom_path = args.ignore - scan_type = 'blacklist' - if not os.path.exists(sbom_path) or not os.path.isfile(sbom_path): - print_stderr(f'Specified --ignore file does not exist or is not a file: {sbom_path}') + if not is_valid_file(args.ignore): exit(1) - if not Scanner.valid_json_file(sbom_path): # Make sure it's a valid JSON file + scan_settings.load_json_file(args.ignore).set_file_type('legacy').set_scan_type( + 'blacklist' + ) + elif args.settings: + if not is_valid_file(args.settings): exit(1) + scan_settings.load_json_file(args.settings).set_file_type('new').set_scan_type( + 'identify' + ) + if args.dep: if not os.path.exists(args.dep) or not os.path.isfile(args.dep): - print_stderr(f'Specified --dep file does not exist or is not a file: {args.dep}') + print_stderr( + f'Specified --dep file does not exist or is not a file: {args.dep}' + ) exit(1) if not Scanner.valid_json_file(args.dep): # Make sure it's a valid JSON file exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: - print_stderr(f'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') + print_stderr( + f'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.' + ) scan_output: str = None if args.output: @@ -563,37 +598,72 @@ def scan(parser, args): print_stderr(f'Using flags {flags}...') elif not args.quiet: if args.timeout < 5: - print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') + print_stderr( + f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.' + ) if args.retry < 0: - print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.') + print_stderr( + f'POST retry (--retry) too small: {args.retry}. Reverting to default.' + ) - if not os.access(os.getcwd(), os.W_OK): # Make sure the current directory is writable. If not disable saving WFP + if not os.access( + os.getcwd(), os.W_OK + ): # Make sure the current directory is writable. If not disable saving WFP print_stderr(f'Warning: Current directory is not writable: {os.getcwd()}') args.no_wfp_output = True if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) pac_file = get_pac_file(args.pac) - scan_options = get_scan_options(args) # Figure out what scanning options we have - - scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, api_key=args.key, url=args.apiurl, - sbom_path=sbom_path, scan_type=scan_type, scan_output=scan_output, output_format=output_format, - flags=flags, nb_threads=args.threads, post_size=args.post_size, - timeout=args.timeout, no_wfp_file=args.no_wfp_output, all_extensions=args.all_extensions, - all_folders=args.all_folders, hidden_files_folders=args.all_hidden, - scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, - grpc_url=args.api2url, obfuscate=args.obfuscate, - ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy, - pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, hpsm=args.hpsm, - skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder, - skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet - ) + scan_options = get_scan_options(args) # Figure out what scanning options we have + + scanner = Scanner( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + api_key=args.key, + url=args.apiurl, + scan_output=scan_output, + output_format=output_format, + flags=flags, + nb_threads=args.threads, + post_size=args.post_size, + timeout=args.timeout, + no_wfp_file=args.no_wfp_output, + all_extensions=args.all_extensions, + all_folders=args.all_folders, + hidden_files_folders=args.all_hidden, + scan_options=scan_options, + sc_timeout=args.sc_timeout, + sc_command=args.sc_command, + grpc_url=args.api2url, + obfuscate=args.obfuscate, + ignore_cert_errors=args.ignore_cert_errors, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + ca_cert=args.ca_cert, + retry=args.retry, + hpsm=args.hpsm, + skip_size=args.skip_size, + skip_extensions=args.skip_extension, + skip_folders=args.skip_folder, + skip_md5_ids=args.skip_md5, + strip_hpsm_ids=args.strip_hpsm, + strip_snippet_ids=args.strip_snippet, + scan_settings=scan_settings + ) + if args.wfp: if not scanner.is_file_or_snippet_scan(): - print_stderr(f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})') + print_stderr( + f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})' + ) exit(1) if scanner.is_dependency_scan() and not args.dep: - print_stderr(f'Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)') + print_stderr( + f'Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)' + ) exit(1) scanner.scan_wfp_with_options(args.wfp, args.dep) elif args.stdin: @@ -601,26 +671,40 @@ def scan(parser, args): if not scanner.scan_contents(args.stdin, contents): exit(1) elif args.files: - if not scanner.scan_files_with_options(args.files, args.dep, scanner.winnowing.file_map): + if not scanner.scan_files_with_options( + args.files, args.dep, scanner.winnowing.file_map + ): exit(1) elif args.scan_dir: if not os.path.exists(args.scan_dir): - print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') + print_stderr( + f'Error: File or folder specified does not exist: {args.scan_dir}.' + ) exit(1) if os.path.isdir(args.scan_dir): - if not scanner.scan_folder_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map): + if not scanner.scan_folder_with_options( + args.scan_dir, args.dep, scanner.winnowing.file_map + ): exit(1) elif os.path.isfile(args.scan_dir): - if not scanner.scan_file_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map): + if not scanner.scan_file_with_options( + args.scan_dir, args.dep, scanner.winnowing.file_map + ): exit(1) else: - print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') + print_stderr( + f'Error: Path specified is neither a file or a folder: {args.scan_dir}.' + ) exit(1) elif args.dep: if not args.dependencies_only: - print_stderr(f'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.') + print_stderr( + f'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.' + ) exit(1) - if not scanner.scan_folder_with_options(".", args.dep, scanner.winnowing.file_map): + if not scanner.scan_folder_with_options( + ".", args.dep, scanner.winnowing.file_map + ): exit(1) else: print_stderr('No action found to process') @@ -707,10 +791,11 @@ def utils_cert_download(_, args): :param _: ignore/unused :param args: Parsed arguments """ - from urllib.parse import urlparse import socket - from OpenSSL import SSL, crypto import traceback + from urllib.parse import urlparse + + from OpenSSL import SSL, crypto file = sys.stdout if args.output: diff --git a/src/scanoss/results.py b/src/scanoss/results.py index dc790149..7174f53a 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -1,7 +1,7 @@ """ SPDX-License-Identifier: MIT - Copyright (c) 2023, SCANOSS + Copyright (c) 2024, SCANOSS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 42574574..a0cec4ba 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -41,11 +41,13 @@ from .scanossgrpc import ScanossGrpc from .scantype import ScanType from .scanossbase import ScanossBase +from .scanoss_settings import ScanossSettings +from .scanpostprocessor import ScanPostProcessor from . import __version__ FAST_WINNOWING = False try: - from scanoss_winnowing.winnowing import Winnowing + from .winnowing import Winnowing FAST_WINNOWING = True except ModuleNotFoundError or ImportError: @@ -95,17 +97,18 @@ class Scanner(ScanossBase): def __init__(self, wfp: str = None, scan_output: str = None, output_format: str = 'plain', debug: bool = False, trace: bool = False, quiet: bool = False, api_key: str = None, url: str = None, - sbom_path: str = None, scan_type: str = None, flags: str = None, nb_threads: int = 5, + flags: str = None, nb_threads: int = 5, post_size: int = 32, timeout: int = 180, no_wfp_file: bool = False, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False, skip_size: int = 0, skip_extensions=None, skip_folders=None, - strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None + strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None, + scan_settings: ScanossSettings = None ): """ - Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning + Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning """ super().__init__(debug, trace, quiet) if skip_folders is None: @@ -133,7 +136,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str skip_md5_ids=skip_md5_ids ) self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, - sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout, + flags=flags, timeout=timeout, ver_details=ver_details, ignore_cert_errors=ignore_cert_errors, proxy=proxy, ca_cert=ca_cert, pac=pac, retry=retry ) @@ -157,6 +160,16 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str if skip_extensions: # Append extra file extensions to skip self.skip_extensions.extend(skip_extensions) + if scan_settings: + self.scan_settings = scan_settings + self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) + self._maybe_set_api_sbom() + + def _maybe_set_api_sbom(self): + sbom = self.scan_settings.get_sbom() + if sbom: + self.scanoss_api.set_sbom(sbom) + def __filter_files(self, files: list) -> list: """ Filter which files should be considered for processing @@ -524,35 +537,24 @@ def __finish_scan_threaded(self, file_map: dict = None) -> bool: raw_output += ",\n \"%s\":[%s]" % (file, json.dumps(dep_file, indent=2)) # End for loop raw_output += "\n}" - parsed_json = None try: - parsed_json = json.loads(raw_output) + raw_results = json.loads(raw_output) except Exception as e: - self.print_stderr(f'Warning: Problem decoding parsed json: {e}') + raise Exception(f'ERROR: Problem decoding parsed json: {e}') + + results = self.post_processor.load_results(raw_results).post_process() if self.output_format == 'plain': - if parsed_json: - self.__log_result(json.dumps(parsed_json, indent=2, sort_keys=True)) - else: - self.__log_result(raw_output) + self.__log_result(json.dumps(results, indent=2, sort_keys=True)) elif self.output_format == 'cyclonedx': cdx = CycloneDx(self.debug, self.scan_output) - if parsed_json: - success = cdx.produce_from_json(parsed_json) - else: - success = cdx.produce_from_str(raw_output) + success = cdx.produce_from_json(results) elif self.output_format == 'spdxlite': spdxlite = SpdxLite(self.debug, self.scan_output) - if parsed_json: - success = spdxlite.produce_from_json(parsed_json) - else: - success = spdxlite.produce_from_str(raw_output) + success = spdxlite.produce_from_json(results) elif self.output_format == 'csv': csvo = CsvOutput(self.debug, self.scan_output) - if parsed_json: - success = csvo.produce_from_json(parsed_json) - else: - success = csvo.produce_from_str(raw_output) + success = csvo.produce_from_json(results) else: self.print_stderr(f'ERROR: Unknown output format: {self.output_format}') success = False @@ -713,7 +715,7 @@ def scan_files(self, files: []) -> bool: else: Scanner.print_stderr(f'Warning: No files found to scan from: {filtered_files}') return success - + def scan_files_with_options(self, files: [], deps_file: str = None, file_map: dict = None) -> bool: """ Scan the given list of files for whatever scaning options that have been configured diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py new file mode 100644 index 00000000..f2c0ad6d --- /dev/null +++ b/src/scanoss/scanoss_settings.py @@ -0,0 +1,198 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the 'Software'), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import os + +from .scanossbase import ScanossBase + + +class ScanossSettings(ScanossBase): + """Handles the loading and parsing of the SCANOSS settings file""" + + def __init__( + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + filepath: str = None, + ): + """ + Args: + debug (bool, optional): Debug. Defaults to False. + trace (bool, optional): Trace. Defaults to False. + quiet (bool, optional): Quiet. Defaults to False. + filepath (str, optional): Path to settings file. Defaults to None. + """ + + super().__init__(debug, trace, quiet) + self.data = {} + self.settings_file_type = None + self.scan_type = None + + if filepath: + self.load_json_file(filepath) + + def load_json_file(self, filepath: str): + """Load the scan settings file + + Args: + filepath (str): Path to the SCANOSS settings file + """ + file = f"{os.getcwd()}/{filepath}" + + if not os.path.exists(file): + self.print_stderr(f"Scan settings file not found: {file}") + self.data = {} + + with open(file, "r") as jsonfile: + self.print_stderr(f"Loading scan settings from: {file}") + try: + self.data = json.load(jsonfile) + except Exception as e: + self.print_stderr(f"ERROR: Problem parsing input JSON: {e}") + return self + + def set_file_type(self, file_type: str): + """Set the file type in order to support both legacy SBOM.json and new scanoss.json files + + Args: + file_type (str): 'legacy' or 'new' + + Raises: + Exception: Invalid scan settings file, missing "components" or "bom" + """ + self.settings_file_type = file_type + if not self._is_valid_sbom_file: + raise Exception( + 'Invalid scan settings file, missing "components" or "bom")' + ) + return self + + def set_scan_type(self, scan_type: str): + """Set the scan type to support legacy SBOM.json files + + Args: + scan_type (str): 'identify' or 'exclude' + """ + self.scan_type = scan_type + return self + + def _is_valid_sbom_file(self): + """Check if the scan settings file is valid + + Returns: + bool: True if the file is valid, False otherwise + """ + if not self.data.get("components") or not self.data.get("bom"): + return False + return True + + def _get_bom(self): + """Get the Billing of Materials from the settings file + + Returns: + dict: If using scanoss.json + list: If using SBOM.json + """ + if self.settings_file_type == "legacy": + return self.data.get("components", []) + return self.data.get("bom", {}) + + def get_bom_include(self): + """Get the list of components to include in the scan + + Returns: + list: List of components to include in the scan + """ + if self.settings_file_type == "legacy": + return self._get_bom() + return self._get_bom().get("include", []) + + def get_bom_remove(self): + """Get the list of components to remove from the scan + + Returns: + list: List of components to remove from the scan + """ + if self.settings_file_type == "legacy": + return self._get_bom() + return self._get_bom().get("remove", []) + + def get_sbom(self): + """Get the SBOM to be sent to the SCANOSS API + + Returns: + dict: SBOM + """ + if not self.data: + return None + return { + "scan_type": self.scan_type, + "assets": json.dumps(self._get_sbom_assets()), + } + + def _get_sbom_assets(self): + """Get the SBOM assets + + Returns: + list: List of SBOM assets + """ + if self.scan_type == "identify": + return self.normalize_bom_entries(self.get_bom_include()) + return self.normalize_bom_entries(self.get_bom_remove()) + + @staticmethod + def normalize_bom_entries(bom_entries): + """Normalize the BOM entries + + Args: + bom_entries (dict): BOM entries + + Returns: + list: Normalized BOM entries + """ + normalized_bom_entries = [] + for entry in bom_entries: + normalized_bom_entries.append( + { + "purl": entry.get("purl", ""), + } + ) + return normalized_bom_entries + + def get_bom_remove_for_filtering(self): + """Get the list of files and purls to remove from the scan + + Returns: + (list[str], list[str]): List of files and list of purls to remove from the scan + """ + entries = self.get_bom_remove() + files = [ + entry.get("path") for entry in entries if entry.get("path") is not None + ] + purls = [ + entry.get("purl") for entry in entries if entry.get("purl") is not None + ] + return files, purls diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 02b151af..3a0643d6 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -34,6 +34,7 @@ from pypac import PACSession from pypac.parser import PACFile from urllib3.exceptions import InsecureRequestWarning + from .scanossbase import ScanossBase from . import __version__ @@ -50,14 +51,12 @@ class ScanossApi(ScanossBase): Currently support posting scan requests to the SCANOSS streaming API """ - def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None, + def __init__(self, scan_format: str = None, flags: str = None, url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, timeout: int = 180, ver_details: str = None, ignore_cert_errors: bool = False, proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5): """ Initialise the SCANOSS API - :param scan_type: Scan type (default identify) - :param sbom_path: Input SBOM file to match scan type (default None) :param scan_format: Scan format (default plain) :param flags: Scanning flags (default None) :param url: API URL (default https://api.osskb.org/scan/direct) @@ -77,9 +76,8 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st self.api_key = api_key if api_key else SCANOSS_API_KEY if self.api_key and not url and not os.environ.get("SCANOSS_SCAN_URL"): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium - self.scan_type = scan_type + self.sbom = None self.scan_format = scan_format if scan_format else 'plain' - self.sbom_path = sbom_path self.flags = flags self.timeout = timeout if timeout > 5 else 180 self.retry_limit = retry if retry >= 0 else 5 @@ -92,8 +90,6 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st self.headers['x-api-key'] = self.api_key self.headers['User-Agent'] = f'scanoss-py/{__version__}' self.headers['user-agent'] = f'scanoss-py/{__version__}' - self.sbom = None - self.load_sbom() # Load an input SBOM if one is specified if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 @@ -115,17 +111,6 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st if self. proxies: self.session.proxies = self.proxies - def load_sbom(self): - """ - Load the input SBOM if one exists - """ - if self.sbom_path: - if not self.scan_type: - self.scan_type = 'identify' # Default to identify SBOM type if it's not set - self.print_debug(f'Loading {self.scan_type} SBOM {self.sbom_path}...') - with open(self.sbom_path) as f: - self.sbom = f.read() - def scan(self, wfp: str, context: str = None, scan_id: int = None): """ Scan the specified WFP and return the JSON object @@ -137,14 +122,15 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): request_id = str(uuid.uuid4()) form_data = {} if self.sbom: - form_data['type'] = self.scan_type - form_data['assets'] = self.sbom + form_data['type'] = self.sbom.get("scan_type") + form_data['assets'] = self.sbom.get("assets") if self.scan_format: form_data['format'] = self.scan_format if self.flags: form_data['flags'] = self.flags if context: form_data['context'] = context + scan_files = {'file': ("%s.wfp" % request_id, wfp)} headers = self.headers headers['x-request-id'] = request_id # send a unique request id for each post @@ -242,6 +228,10 @@ def save_bad_req_wfp(self, scan_files, request_id, scan_id): except Exception as ee: self.print_stderr(f'Warning: Issue writing bad request file - {bad_req_file} ({ee.__class__.__name__}):' f' {ee}') + + def set_sbom(self, sbom): + self.sbom = sbom + return self # # End of ScanossApi Class diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py new file mode 100644 index 00000000..8c5c88ff --- /dev/null +++ b/src/scanoss/scanpostprocessor.py @@ -0,0 +1,104 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +from .scanoss_settings import ScanossSettings +from .scanossbase import ScanossBase + + +class ScanPostProcessor(ScanossBase): + """Handles post-processing of the scan results""" + + def __init__( + self, + scan_settings: ScanossSettings, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + results: dict = None, + ): + """ + Args: + scan_settings (ScanossSettings): Scan settings object + debug (bool, optional): Debug mode. Defaults to False. + trace (bool, optional): Traces. Defaults to False. + quiet (bool, optional): Quiet mode. Defaults to False. + results (dict | str, optional): Results to be processed. Defaults to None. + """ + super().__init__(debug, trace, quiet) + self.scan_settings = scan_settings + self.results = results + + def load_results(self, raw_results: dict): + """Load the raw results + + Args: + raw_results (dict): Raw scan results + """ + self.results = raw_results + return self + + def post_process(self): + """Post-process the scan results + + Returns: + dict: Processed results + """ + self.remove_dismissed_files() + return self.results + + def remove_dismissed_files(self): + """Remove dismissed files in SCANOSS settings file from the results""" + to_remove_files, to_remove_purls = ( + self.scan_settings.get_bom_remove_for_filtering() + ) + + if not to_remove_files and not to_remove_purls: + return + + self.filter_files(to_remove_files, to_remove_purls) + return self + + def filter_files(self, files: list, purls: list): + """Filter files based on the provided list of files and purls + + Args: + files (list): List of files to be filtered + purls (list): List of purls to be filtered + """ + filtered_results = {} + + for file_name in self.results: + file = self.results.get(file_name) + file = file[0] if isinstance(file, list) else file + + identified_purls = file.get("purl") + if identified_purls and any(purl in purls for purl in identified_purls): + continue + elif file_name in files: + continue + + filtered_results[file_name] = file + + self.results = filtered_results + return self diff --git a/tests/scanpostprocessor-test.py b/tests/scanpostprocessor-test.py new file mode 100644 index 00000000..58f8d72e --- /dev/null +++ b/tests/scanpostprocessor-test.py @@ -0,0 +1,60 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import unittest + +from src.scanoss.scanoss_settings import ScanossSettings +from src.scanoss.scanpostprocessor import ScanPostProcessor + + +class MyTestCase(unittest.TestCase): + """ + Unit test cases for Scan Post-Processing + """ + + def test_remove_files(self): + """ + Should remove files by path from the scan results + """ + scan_settings = ScanossSettings(filepath="data/scanoss.json") + post_processor = ScanPostProcessor(scan_settings) + results = { + "scanoss_settings.py": [ + { + "purl": ["pkg:github/scanoss/scanoss.py"], + } + ], + "test_file_path.go": [ + { + "purl": ["pkg:github/scanoss/scanoss.lui"], + } + ] + } + processed_results = post_processor.load_results(results).post_process() + + self.assertEqual(len(processed_results), 0) + self.assertEqual(processed_results, {}) + + +if __name__ == '__main__': + unittest.main() From 718bbda3814e7d6a9b9d388f2d481d0a3130b29c Mon Sep 17 00:00:00 2001 From: Lucas Gonze Date: Mon, 16 Sep 2024 04:29:51 -0400 Subject: [PATCH 165/489] Fix https://github.com/scanoss/scanoss.py/issues/48: add documentation (#55) Signed-off-by: Lucas Gonze --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7280b019..bb640bec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ RUN mkdir /install WORKDIR /install ENV PATH=/root/.local/bin:$PATH +# assumes `make dist` as prerequisite COPY ./dist/scanoss-*-py3-none-any.whl /install/ # Install dependencies From d6ba0c035a26473c80a6f1cab0dcef4bc4864555 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 16 Sep 2024 17:59:03 +0200 Subject: [PATCH 166/489] chore: scanoss results post processor enhancements Added debug messages Handle non relative paths for settings file Only remove results if matches both path and purl (if present) --- src/scanoss/cli.py | 7 +- src/scanoss/scanoss_settings.py | 45 +++++------ src/scanoss/scanpostprocessor.py | 128 +++++++++++++++++++++++++------ tests/scanpostprocessor-test.py | 53 +++++++++++-- 4 files changed, 171 insertions(+), 62 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 08ddd6b3..cfff4882 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -23,6 +23,7 @@ """ import argparse import os +from pathlib import Path import sys import pypac @@ -1021,9 +1022,9 @@ def results(parser, args): parser.parse_args([args.subparser, "-h"]) exit(1) - results_file = f"{os.getcwd()}/{args.filepath}" + file_path = Path(args.filepath).resolve() - if not os.path.isfile(results_file): + if not file_path.is_file(): print_stderr(f"The specified file {args.filepath} does not exist") exit(1) @@ -1031,7 +1032,7 @@ def results(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - filepath=results_file, + filepath=file_path, match_type=args.match_type, status=args.status, output_file=args.output, diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index f2c0ad6d..e8aad44e 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -23,11 +23,17 @@ """ import json -import os +from pathlib import Path +from typing import Dict, List, TypedDict from .scanossbase import ScanossBase +class BomEntry(TypedDict, total=False): + purl: str + path: str + + class ScanossSettings(ScanossBase): """Handles the loading and parsing of the SCANOSS settings file""" @@ -60,14 +66,14 @@ def load_json_file(self, filepath: str): Args: filepath (str): Path to the SCANOSS settings file """ - file = f"{os.getcwd()}/{filepath}" + json_file = Path(filepath).resolve() - if not os.path.exists(file): - self.print_stderr(f"Scan settings file not found: {file}") + if not json_file.exists(): + self.print_stderr(f"Scan settings file not found: {filepath}") self.data = {} - with open(file, "r") as jsonfile: - self.print_stderr(f"Loading scan settings from: {file}") + with open(json_file, "r") as jsonfile: + self.print_debug(f"Loading scan settings from: {filepath}") try: self.data = json.load(jsonfile) except Exception as e: @@ -120,7 +126,7 @@ def _get_bom(self): return self.data.get("components", []) return self.data.get("bom", {}) - def get_bom_include(self): + def get_bom_include(self) -> List[BomEntry]: """Get the list of components to include in the scan Returns: @@ -130,7 +136,7 @@ def get_bom_include(self): return self._get_bom() return self._get_bom().get("include", []) - def get_bom_remove(self): + def get_bom_remove(self) -> List[BomEntry]: """Get the list of components to remove from the scan Returns: @@ -157,21 +163,21 @@ def _get_sbom_assets(self): """Get the SBOM assets Returns: - list: List of SBOM assets + List: List of SBOM assets """ if self.scan_type == "identify": return self.normalize_bom_entries(self.get_bom_include()) return self.normalize_bom_entries(self.get_bom_remove()) @staticmethod - def normalize_bom_entries(bom_entries): + def normalize_bom_entries(bom_entries) -> List[BomEntry]: """Normalize the BOM entries Args: - bom_entries (dict): BOM entries + bom_entries (List[Dict]): List of BOM entries Returns: - list: Normalized BOM entries + List: Normalized BOM entries """ normalized_bom_entries = [] for entry in bom_entries: @@ -181,18 +187,3 @@ def normalize_bom_entries(bom_entries): } ) return normalized_bom_entries - - def get_bom_remove_for_filtering(self): - """Get the list of files and purls to remove from the scan - - Returns: - (list[str], list[str]): List of files and list of purls to remove from the scan - """ - entries = self.get_bom_remove() - files = [ - entry.get("path") for entry in entries if entry.get("path") is not None - ] - purls = [ - entry.get("purl") for entry in entries if entry.get("purl") is not None - ] - return files, purls diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 8c5c88ff..99e1e3f9 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -22,7 +22,9 @@ THE SOFTWARE. """ -from .scanoss_settings import ScanossSettings +from typing import List + +from .scanoss_settings import BomEntry, ScanossSettings from .scanossbase import ScanossBase @@ -68,37 +70,115 @@ def post_process(self): return self.results def remove_dismissed_files(self): - """Remove dismissed files in SCANOSS settings file from the results""" - to_remove_files, to_remove_purls = ( - self.scan_settings.get_bom_remove_for_filtering() - ) + """Remove entries from the results based on files and/or purls specified in the SCANOSS settings file""" + + to_remove_entries = self.scan_settings.get_bom_remove() - if not to_remove_files and not to_remove_purls: + if not to_remove_entries: return - self.filter_files(to_remove_files, to_remove_purls) - return self + self.results = { + result_path: result + for result_path, result in self.results.items() + if not self._should_remove_result(result_path, result, to_remove_entries) + } + + def _should_remove_result( + self, result_path: str, result: dict, to_remove_entries: List[BomEntry] + ) -> bool: + """Check if a result should be removed based on the SCANOSS settings""" + result = result[0] if isinstance(result, list) else result + result_purls = result.get("purl", []) + + for to_remove_entry in to_remove_entries: + to_remove_path = to_remove_entry.get("path") + to_remove_purl = to_remove_entry.get("purl") + + if not to_remove_path and not to_remove_purl: + continue + + if to_remove_path and to_remove_purl: + if self._is_full_match(result_path, result_purls, to_remove_entry): + self._print_removal_message( + result_path, result_purls, to_remove_entry + ) + return True + elif self._is_partial_match(result_path, result_purls, to_remove_entry): + self._print_removal_message( + result_path, result_purls, to_remove_entry + ) + return True + elif to_remove_purl and not to_remove_path: + if to_remove_purl in result_purls: + self._print_removal_message( + result_path, result_purls, to_remove_entry + ) + return True + elif to_remove_path and not to_remove_purl: + if to_remove_path == result_path: + self._print_removal_message( + result_path, result_purls, to_remove_entry + ) + return True + + return False + + def _print_removal_message( + self, result_path: str, result_purls: List[str], to_remove_entry: BomEntry + ) -> None: + """Print a message about removing a result""" + if to_remove_entry.get("path") and to_remove_entry.get("purl"): + message = f"Removing '{result_path}' from the results. Full match found." + elif to_remove_entry.get("purl"): + message = f"Removing '{result_path}' from the results. Found PURL match." + else: + message = f"Removing '{result_path}' from the results. Found path match." + + self.print_msg( + f"{message}\n" + f"Details:\n" + f" - PURLs: {', '.join(result_purls)}\n" + f" - Path: '{result_path}'\n" + ) - def filter_files(self, files: list, purls: list): - """Filter files based on the provided list of files and purls + def _is_full_match( + self, + result_path: str, + result_purls: List[str], + bom_entry: BomEntry, + ) -> bool: + """Check if path and purl matches fully with the bom entry Args: - files (list): List of files to be filtered - purls (list): List of purls to be filtered + result_path (str): Scan result path + result_purls (List[str]): Scan result purls + bom_entry (BomEntry): BOM entry to compare with + + Returns: + bool: True if the path and purl match, False otherwise """ - filtered_results = {} + purl = bom_entry.get("purl") + path = bom_entry.get("path") + if not result_purls: + return + return purl and path and path == result_path and purl in result_purls - for file_name in self.results: - file = self.results.get(file_name) - file = file[0] if isinstance(file, list) else file + def _is_partial_match( + self, result_path: str, result_purls: List[str], bom_entry: BomEntry + ) -> bool: + """Check if path or purl matches partially with the BOM entry - identified_purls = file.get("purl") - if identified_purls and any(purl in purls for purl in identified_purls): - continue - elif file_name in files: - continue + Args: + result_path (str): Scan result path + result_purls (str): Scan result purl + bom_entry (BomEntry): BOM entry to compare with - filtered_results[file_name] = file + Returns: + bool: True if the path and purl match, False otherwise + """ - self.results = filtered_results - return self + purl = bom_entry.get("purl") + path = bom_entry.get("path") + # if not result_purls: + # return False + return path and path == result_path or purl and purl in result_purls diff --git a/tests/scanpostprocessor-test.py b/tests/scanpostprocessor-test.py index 58f8d72e..b462848c 100644 --- a/tests/scanpostprocessor-test.py +++ b/tests/scanpostprocessor-test.py @@ -21,10 +21,11 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + import unittest -from src.scanoss.scanoss_settings import ScanossSettings -from src.scanoss.scanpostprocessor import ScanPostProcessor +from scanoss.scanoss_settings import ScanossSettings +from scanoss.scanpostprocessor import ScanPostProcessor class MyTestCase(unittest.TestCase): @@ -32,26 +33,62 @@ class MyTestCase(unittest.TestCase): Unit test cases for Scan Post-Processing """ + scan_settings = ScanossSettings(filepath="tests/data/scanoss.json") + post_processor = ScanPostProcessor(scan_settings) + def test_remove_files(self): """ - Should remove files by path from the scan results + Should remove component if matches path and purl """ - scan_settings = ScanossSettings(filepath="data/scanoss.json") - post_processor = ScanPostProcessor(scan_settings) + results = { "scanoss_settings.py": [ { "purl": ["pkg:github/scanoss/scanoss.py"], } ], + } + processed_results = self.post_processor.load_results(results).post_process() + + self.assertEqual(len(processed_results), 0) + self.assertEqual(processed_results, {}) + + def test_remove_files_no_results(self): + """ + Should return empty dictionary when empty results are provided + """ + processed_results = self.post_processor.load_results({}).post_process() + + self.assertEqual(len(processed_results), 0) + self.assertEqual(processed_results, {}) + + def test_remove_files_path_match(self): + """ + Should remove component if matches path + """ + results = { "test_file_path.go": [ { - "purl": ["pkg:github/scanoss/scanoss.lui"], + "purl": ["no/matching/purl"], } - ] + ], } - processed_results = post_processor.load_results(results).post_process() + processed_results = self.post_processor.load_results(results).post_process() + self.assertEqual(len(processed_results), 0) + self.assertEqual(processed_results, {}) + def test_remove_files_purl_match(self): + """ + Should remove component if matches purl + """ + results = { + "no_matching_path.go": [ + { + "purl": ["matching/purl"], + } + ], + } + processed_results = self.post_processor.load_results(results).post_process() self.assertEqual(len(processed_results), 0) self.assertEqual(processed_results, {}) From b7dc03b48559dfd9883432d85c961875ec5895bd Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 17 Sep 2024 10:36:24 +0200 Subject: [PATCH 167/489] chore: scanoss results post processor enhancements Fix full match (path + purl) filtering --- src/scanoss/scanpostprocessor.py | 67 ++++++++++---------------------- 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 99e1e3f9..6f1e9b0e 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -97,29 +97,20 @@ def _should_remove_result( if not to_remove_path and not to_remove_purl: continue - if to_remove_path and to_remove_purl: - if self._is_full_match(result_path, result_purls, to_remove_entry): - self._print_removal_message( - result_path, result_purls, to_remove_entry - ) - return True - elif self._is_partial_match(result_path, result_purls, to_remove_entry): - self._print_removal_message( - result_path, result_purls, to_remove_entry - ) - return True - elif to_remove_purl and not to_remove_path: - if to_remove_purl in result_purls: - self._print_removal_message( - result_path, result_purls, to_remove_entry - ) - return True - elif to_remove_path and not to_remove_purl: - if to_remove_path == result_path: - self._print_removal_message( - result_path, result_purls, to_remove_entry - ) - return True + # Bom entry has both path and purl + if self._is_full_match(result_path, result_purls, to_remove_entry): + self._print_removal_message(result_path, result_purls, to_remove_entry) + return True + + # Bom entry has only purl + if not to_remove_path and to_remove_purl in result_purls: + self._print_removal_message(result_path, result_purls, to_remove_entry) + return True + + # Bom entry has only path + if not to_remove_purl and to_remove_path == result_path: + self._print_removal_message(result_path, result_purls, to_remove_entry) + return True return False @@ -157,28 +148,12 @@ def _is_full_match( Returns: bool: True if the path and purl match, False otherwise """ - purl = bom_entry.get("purl") - path = bom_entry.get("path") - if not result_purls: - return - return purl and path and path == result_path and purl in result_purls - def _is_partial_match( - self, result_path: str, result_purls: List[str], bom_entry: BomEntry - ) -> bool: - """Check if path or purl matches partially with the BOM entry - - Args: - result_path (str): Scan result path - result_purls (str): Scan result purl - bom_entry (BomEntry): BOM entry to compare with - - Returns: - bool: True if the path and purl match, False otherwise - """ + if not result_purls: + return False - purl = bom_entry.get("purl") - path = bom_entry.get("path") - # if not result_purls: - # return False - return path and path == result_path or purl and purl in result_purls + return bool( + (bom_entry.get("purl") and bom_entry.get("path")) + and (bom_entry.get("path") == result_path) + and (bom_entry.get("purl") in result_purls) + ) From 29dbb2f96d7322eed4e1d587ffdf8285c844608a Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 17 Sep 2024 11:56:17 +0200 Subject: [PATCH 168/489] chore: update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27c19d80..37fad99a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.15.0] - 2024-09-03 +## [1.15.0] - 2024-09-17 ### Added - Added Results sub-command: - Get all results (`scanoss-py results /path/to/file`) - Get filtered results (`scanoss-py results /path/to/file --match-type=file,snippet status=pending`) - Get pending declarations (`scanoss-py results /path/to/file --has-pending`) +- Added `--settings` option to `scan` command to specify a settings file + - Specify settings file (`scanoss-py scan --settings /path/to/settings.json /path/to/file`) ## [1.14.0] - 2024-08-09 ### Added From 694e1393e709690ce3964ec03c5469d0f2ceb7c4 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Tue, 17 Sep 2024 07:41:52 -0300 Subject: [PATCH 169/489] Feat/SP-1423 filter dependencies based on their scopes * feat: SP-1423 Filters dependencies based on their scopes * feat: SP-1439 Adds option to include or exclude dependencies base on custom scopes * chore: SP-14441 Adds units tests for dependency scope filtering feature * chore: SP-1484 Refactor on dependency scope filter * chore: SP-1493 Add dependency scope filtering documentation * fix: SP-1496 Fixes dep scan and adds dep scan integration test --------- Co-authored-by: eeisegn --- CHANGELOG.md | 12 +- CLIENT_HELP.md | 21 +++ src/scanoss/cli.py | 24 ++-- src/scanoss/scancodedeps.py | 32 ++++- src/scanoss/scanner.py | 20 ++- src/scanoss/threadeddependencies.py | 80 ++++++++++- tests/data/package.json | 4 + tests/scancodedeps-test.py | 197 +++++++++++++++++++++++++++- 8 files changed, 357 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37fad99a..337454bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,16 +12,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.15.0] - 2024-09-17 ### Added - Added Results sub-command: - - Get all results (`scanoss-py results /path/to/file`) - - Get filtered results (`scanoss-py results /path/to/file --match-type=file,snippet status=pending`) - - Get pending declarations (`scanoss-py results /path/to/file --has-pending`) +- Get all results (`scanoss-py results /path/to/file`) +- Get filtered results (`scanoss-py results /path/to/file --match-type=file,snippet status=pending`) +- Get pending declarations (`scanoss-py results /path/to/file --has-pending`) - Added `--settings` option to `scan` command to specify a settings file - - Specify settings file (`scanoss-py scan --settings /path/to/settings.json /path/to/file`) +- Specify settings file (`scanoss-py scan --settings /path/to/settings.json /path/to/file`) +- Added support for filtering dependencies based on development or production dependency scopes +- Added support for defining custom scopes to include or exclude dependencies with specified scope criteria ## [1.14.0] - 2024-08-09 ### Added - Added support for Python3.12 - - Module `pkg_resources` has been replaced with `importlib_resources` +- Module `pkg_resources` has been replaced with `importlib_resources` - Added support for UTF-16 filenames ## [1.13.0] - 2024-06-05 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 1f5301c0..b5cde885 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -189,6 +189,27 @@ The following command scans the `src` folder for file, snippet & dependency matc scanoss-py scan -o scan-results.json -D src ``` +### Scan a project folder with filtered dependency scopes +The following command scans the src folder for files, code snippets, and dependencies, specifically targeting development dependencies: +The available flags for filtering dependency scopes are **__dev__** for development dependencies or **__prod__** for production dependencies: +```bash +scanoss-py scan -D src --dep-scope dev +``` + +### Scan a project folder including dependencies with declared scopes +The following command scans the src folder for files, code snippets, and dependencies, allowing you to specify which dependency scopes to include. +In this example, the scan targets the dependencies and install scopes: +```bash +scanoss-py scan -D src --dep-scope-inc dependencies,install +``` + +### Scan a project folder excluding dependencies with declared scopes +The following command scans the src folder for files, code snippets, and dependencies, allowing you to specify which dependency scopes to exclude. +In this example, the scan targets dependencies but excludes those within the install scope: +```bash +scanoss-py scan -D src --dep-scope-exc install +``` + ### Scan a project folder skipping files and snippets The following command scans the `src` folder writing the output to `scan-results.json` skipping the following: - MD5 file `37f7cd1e657aa3c30ece35995b4c59e5` diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index cfff4882..8bc73475 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -25,10 +25,11 @@ import os from pathlib import Path import sys +from array import array import pypac - +from .threadeddependencies import SCOPE from .scanner import Scanner from .scanoss_settings import ScanossSettings from .scancodedeps import ScancodeDeps @@ -105,12 +106,16 @@ def setup_args() -> None: help='Scancode command and path if required (optional - default scancode).') p_scan.add_argument('--sc-timeout', type=int, default=600, help='Timeout (in seconds) for scancode to complete (optional - default 600)') + p_scan.add_argument('--dep-scope', '-ds', type=SCOPE, help='Filter dependencies by scope - default all (options: dev/prod)') + p_scan.add_argument('--dep-scope-inc', '-dsi', type=str,help='Include dependencies with declared scopes') + p_scan.add_argument('--dep-scope-exc', '-dse', type=str, help='Exclude dependencies with declared scopes') p_scan.add_argument( '--settings', type=str, help='Settings file to use for scanning (optional - default scanoss.json)', ) + # Sub-command: fingerprint p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'], description=f'Fingerprint the given source base: {__version__}', @@ -654,7 +659,7 @@ def is_valid_file(file_path: str) -> bool: strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings ) - + if args.wfp: if not scanner.is_file_or_snippet_scan(): print_stderr( @@ -683,14 +688,12 @@ def is_valid_file(file_path: str) -> bool: ) exit(1) if os.path.isdir(args.scan_dir): - if not scanner.scan_folder_with_options( - args.scan_dir, args.dep, scanner.winnowing.file_map - ): + if not scanner.scan_folder_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map, + args.dep_scope, args.dep_scope_inc, args.dep_scope_exc): exit(1) elif os.path.isfile(args.scan_dir): - if not scanner.scan_file_with_options( - args.scan_dir, args.dep, scanner.winnowing.file_map - ): + if not scanner.scan_file_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map, + args.dep_scope, args.dep_scope_inc, args.dep_scope_exc): exit(1) else: print_stderr( @@ -703,9 +706,8 @@ def is_valid_file(file_path: str) -> bool: f'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.' ) exit(1) - if not scanner.scan_folder_with_options( - ".", args.dep, scanner.winnowing.file_map - ): + if not scanner.scan_folder_with_options(".", args.dep, scanner.winnowing.file_map,args.dep_scope, + args.dep_scope_inc, args.dep_scope_exc): exit(1) else: print_stderr('No action found to process') diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index e1b1bebf..7527d0fd 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -59,6 +59,7 @@ def __log_result(self, string, outfile=None): else: print(string) + def remove_interim_file(self, output_file: str = None): """ Remove the temporary Scancode interim file @@ -105,15 +106,17 @@ def produce_from_json(self, data: json) -> dict: continue self.print_debug(f'Path: {f_path}, Packages: {len(f_packages)}') purls = [] + scopes = [] for pkgs in f_packages: pk_deps = pkgs.get('dependencies') + if not pk_deps or pk_deps == '': continue - self.print_debug(f'Path: {f_path}, Dependencies: {len(pk_deps)}') for d in pk_deps: dp = d.get('purl') if not dp or dp == '': continue + dp = dp.replace('"', '').replace('%22', '') # remove unwanted quotes on purls dp_data = {'purl': dp} rq = d.get('extracted_requirement') # scancode format 2.0 @@ -122,15 +125,21 @@ def produce_from_json(self, data: json) -> dict: # skip requirement if it ends with the purl (i.e. exact version) or if it's local (file) if rq and rq != '' and not dp.endswith(rq) and not rq.startswith('file:'): dp_data['requirement'] = rq + + # Gets dependency scope + scope = d.get('scope') + if scope and scope != '': + dp_data['scope'] = scope + purls.append(dp_data) - # self.print_stderr(f'Path: {f_path}, Purls: {purls}') + # end for loop + if len(purls) > 0: files.append({'file': f_path, 'purls': purls}) # End packages # End file details # End dependencies json deps = {'files': files} - # self.print_debug(f'Dep Data: {deps}') return deps def produce_from_file(self, json_file: str = None) -> json: @@ -179,6 +188,7 @@ def get_dependencies(self, output_file: str = None, what_to_scan: str = None, re return False self.print_msg('Producing summary...') deps = self.produce_from_file(output_file) + deps = self.__remove_dep_scope(deps) self.remove_interim_file(output_file) if not deps: return False @@ -235,6 +245,22 @@ def load_from_file(self, json_file: str = None) -> json: self.print_stderr(f'ERROR: Problem loading input JSON: {e}') return None + + @staticmethod + def __remove_dep_scope(deps: json)->json: + """ + :param deps: dependencies with scopes + :return dependencies without scopes + """ + files = deps.get("files") + for file in files: + if 'purls' in file: + purls = file.get("purls") + for purl in purls: + purl.pop("scope",None) + + return {"files": files } + # # End of ScancodeDeps Class # diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index a0cec4ba..f052ee07 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -37,7 +37,7 @@ from .csvoutput import CsvOutput from .threadedscanning import ThreadedScanning from .scancodedeps import ScancodeDeps -from .threadeddependencies import ThreadedDependencies +from .threadeddependencies import ThreadedDependencies, SCOPE from .scanossgrpc import ScanossGrpc from .scantype import ScanType from .scanossbase import ScanossBase @@ -342,14 +342,20 @@ def is_dependency_scan(self): return True return False - def scan_folder_with_options(self, scan_dir: str, deps_file: str = None, file_map: dict = None) -> bool: + def scan_folder_with_options(self, scan_dir: str, deps_file: str = None, file_map: dict = None, + dep_scope: SCOPE = None, dep_scope_include: str = None, + dep_scope_exclude: str = None) -> bool: """ Scan the given folder for whatever scaning options that have been configured + :param dep_scope_exclude: comma separated list of dependency scopes to exclude + :param dep_scope_include: comma separated list of dependency scopes to include + :param dep_scope: Enum dependency scope to use :param scan_dir: directory to scan :param deps_file: pre-parsed dependency file to decorate :param file_map: mapping of obfuscated files back into originals :return: True if successful, False otherwise """ + success = True if not scan_dir: raise Exception(f"ERROR: Please specify a folder to scan") @@ -361,7 +367,8 @@ def scan_folder_with_options(self, scan_dir: str, deps_file: str = None, file_ma if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') if self.is_dependency_scan(): - if not self.threaded_deps.run(what_to_scan=scan_dir, deps_file=deps_file, wait=False): # Kick off a background dependency scan + if not self.threaded_deps.run(what_to_scan=scan_dir, deps_file=deps_file, wait=False, dep_scope=dep_scope, + dep_scope_include= dep_scope_include, dep_scope_exclude=dep_scope_exclude): # Kick off a background dependency scan success = False if self.is_file_or_snippet_scan(): if not self.scan_folder(scan_dir): @@ -560,9 +567,11 @@ def __finish_scan_threaded(self, file_map: dict = None) -> bool: success = False return success - def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dict = None) -> bool: + def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dict = None, dep_scope: SCOPE = None, + dep_scope_include: str = None, dep_scope_exclude: str = None) -> bool: """ Scan the given file for whatever scaning options that have been configured + :param dep_scope: :param file: file to scan :param deps_file: pre-parsed dependency file to decorate :param file_map: mapping of obfuscated files back into originals @@ -579,7 +588,8 @@ def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dic if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') if self.is_dependency_scan(): - if not self.threaded_deps.run(what_to_scan=file, deps_file=deps_file, wait=False): # Kick off a background dependency scan + if not self.threaded_deps.run(what_to_scan=file, deps_file=deps_file, wait=False, dep_scope=dep_scope, + dep_scope_include=dep_scope_include, dep_scope_exclude=dep_scope_exclude): # Kick off a background dependency scan success = False if self.is_file_or_snippet_scan(): if not self.scan_file(file): diff --git a/src/scanoss/threadeddependencies.py b/src/scanoss/threadeddependencies.py index 289fb6a9..036c6831 100644 --- a/src/scanoss/threadeddependencies.py +++ b/src/scanoss/threadeddependencies.py @@ -24,7 +24,9 @@ import threading import queue -from typing import Dict +import json +from enum import Enum +from typing import Dict, Optional, Set from dataclasses import dataclass from .scancodedeps import ScancodeDeps @@ -33,6 +35,14 @@ DEP_FILE_PREFIX = "file=" # Default prefix to signify an existing parsed dependency file +DEV_DEPENDENCIES = { "dev", "test", "development", "provided", "runtime", "devDependencies", "dev-dependencies", "testImplementation", "testCompile", "Test", "require-dev" } + + +# Define an enum class +class SCOPE(Enum): + PRODUCTION = 'prod' + DEVELOPMENT = 'dev' + @dataclass class ThreadedDependencies(ScanossBase): @@ -66,9 +76,13 @@ def responses(self) -> Dict: return resp return None - def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True) -> bool: + def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True, dep_scope: SCOPE = None, + dep_scope_include: str = None, dep_scope_exclude: str = None) -> bool: """ Initiate a background scan for the specified file/dir + :param dep_scope_exclude: comma separated list of dependency scopes to exclude + :param dep_scope_include: comma separated list of dependency scopes to include + :param dep_scope: Enum dependency scope to use :param what_to_scan: file/folder to scan :param deps_file: file to decorate instead of scan (overrides what_to_scan option) :param wait: wait for completion @@ -82,8 +96,9 @@ def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}') # Add to queue and have parent wait on it else: # Search for dependencies to decorate self.print_msg(f'Searching {what_to_scan} for dependencies...') - self.inputs.put(what_to_scan) # Add to queue and have parent wait on it - self._thread = threading.Thread(target=self.scan_dependencies, daemon=True) + self.inputs.put(what_to_scan) + # Add to queue and have parent wait on it + self._thread = threading.Thread(target=self.scan_dependencies(dep_scope, dep_scope_include, dep_scope_exclude), daemon=True) self._thread.start() except Exception as e: self.print_stderr(f'ERROR: Problem running threaded dependencies: {e}') @@ -92,7 +107,54 @@ def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True self.complete() return False if self._errors else True - def scan_dependencies(self) -> None: + def filter_dependencies(self,deps ,filter_dep)-> json: + files = deps.get('files', []) + # Iterate over files and their purls + for file in files: + if 'purls' in file: + # Filter purls with scope 'dependencies' and remove the scope field + file['purls'] = [ + {key: value for key, value in purl.items() if key != 'scope'} + for purl in file['purls'] + if filter_dep(purl.get('scope')) + ] + # End of for loop + + return { + 'files': [ + file for file in deps.get('files', []) + if file.get('purls') + ] + } + + def filter_dependencies_by_scopes(self,deps: json, dep_scope: SCOPE = None, dep_scope_include: str = None, + dep_scope_exclude: str = None) -> json: + # Predefined set of scopes to filter + + # Include all scopes + include_all = (dep_scope is None or dep_scope == "") and dep_scope_include is None and dep_scope_exclude is None + ## All dependencies, remove scope key + if include_all: + return self.filter_dependencies(deps, lambda purl:True) + + # Use default list of scopes if a custom list is not set + if (dep_scope is not None and dep_scope != "") and dep_scope_include is None and dep_scope_exclude is None: + return self.filter_dependencies(deps, lambda purl: (dep_scope == SCOPE.PRODUCTION and purl not in DEV_DEPENDENCIES) or + dep_scope == SCOPE.DEVELOPMENT and purl in DEV_DEPENDENCIES) + + if ((dep_scope_include is not None and dep_scope_include != "") + or dep_scope_exclude is not None and dep_scope_exclude != ""): + # Create sets from comma-separated strings, if provided + exclude = set(dep_scope_exclude.split(',')) if dep_scope_exclude else set() + include = set(dep_scope_include.split(',')) if dep_scope_include else set() + + # Define a lambda function that checks the inclusion/exclusion logic + return self.filter_dependencies( + deps, + lambda purl: (exclude and purl not in exclude) or (not exclude and purl in include) + ) + + def scan_dependencies(self, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None) -> None: """ Scan for dependencies from the given file/dir or from an input file (from the input queue). """ @@ -108,6 +170,14 @@ def scan_dependencies(self) -> None: self._errors = True else: deps = self.sc_deps.produce_from_file() + if dep_scope is not None: + self.print_debug(f'Filtering {dep_scope.name} dependencies') + if dep_scope_include is not None: + self.print_debug(f"Including dependencies with '{dep_scope_include.split(',')}' scopes") + if dep_scope_exclude is not None: + self.print_debug(f"Excluding dependencies with '{dep_scope_exclude.split(',')}' scopes") + deps = self.filter_dependencies_by_scopes(deps, dep_scope,dep_scope_include, dep_scope_exclude) + if not self._errors: if deps is None: self.print_stderr(f'Problem searching for dependencies for: {what_to_scan}') diff --git a/tests/data/package.json b/tests/data/package.json index ff143968..6fb162a1 100644 --- a/tests/data/package.json +++ b/tests/data/package.json @@ -15,6 +15,10 @@ "author": "Evan You", "license": "MIT", "homepage": "https://github.com/vuejs/vue#readme", + "dependencies": { + "uuid": "^9.0.0", + "xml-js": "^1.6.11" + }, "devDependencies": { "@babel/core": ">0.2.0" } diff --git a/tests/scancodedeps-test.py b/tests/scancodedeps-test.py index 90a37c2b..59811b85 100644 --- a/tests/scancodedeps-test.py +++ b/tests/scancodedeps-test.py @@ -21,19 +21,21 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import json import os +import tempfile import unittest -from scanoss.scancodedeps import ScancodeDeps -from scanoss.scanossgrpc import ScanossGrpc -from scanoss.threadeddependencies import ThreadedDependencies +from src.scanoss.scancodedeps import ScancodeDeps +from src.scanoss.scanossgrpc import ScanossGrpc +from src.scanoss.threadeddependencies import ThreadedDependencies, SCOPE class MyTestCase(unittest.TestCase): """ Unit test cases for Scancode Dependency analysis """ - TEST_LOCAL = os.getenv("SCANOSS_TEST_LOCAL", 'True').lower() in ('true', '1', 't', 'yes', 'y') + TEST_LOCAL = os.getenv("SCANOSS_TEST_LOCAL", 'False').lower() in ('true', '1', 't', 'yes', 'y') def test_deps_parse(self): """ @@ -76,6 +78,193 @@ def test_threaded_scan_dir(self): print(f'Dependency results ({server_type}): {deps}') self.assertIsNotNone(deps) + def test_dep_scope_all(self): + """ + Run a dependency scan of the current directory, then parse those results + """ + # with open('scanoss-com.pem', 'rb') as f: + # root_certs = f.read() + if MyTestCase.TEST_LOCAL: + server_type = "local" + grpc_client = ScanossGrpc(debug=True, url='localhost:50051') + else: + server_type = "remote" + grpc_client = ScanossGrpc(debug=True) + sc_deps = ScancodeDeps(debug=True) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True)) + deps = threaded_deps.responses + files = deps.get("files") + package_json_deps = files[0]["dependencies"] + requirements_txt_deps = files[1].get("dependencies", []) + print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') + print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') + self.assertEqual(len(package_json_deps),3) + self.assertEqual(len(requirements_txt_deps), 6) + + + def test_dep_scope_development(self): + """ + Run a dependency scan of the current directory, then parse those results + """ + # with open('scanoss-com.pem', 'rb') as f: + # root_certs = f.read() + if MyTestCase.TEST_LOCAL: + server_type = "local" + grpc_client = ScanossGrpc(debug=True, url='localhost:50051') + else: + server_type = "remote" + grpc_client = ScanossGrpc(debug=True) + sc_deps = ScancodeDeps(debug=True) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope=SCOPE.DEVELOPMENT)) + deps = threaded_deps.responses + files = deps.get("files") + package_json_dev_deps = files[0]["dependencies"] + requirements_txt_dev_deps = files[1].get("dependencies", []) + print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_dev_deps}') + print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_dev_deps}') + self.assertNotEquals(len(package_json_dev_deps),len(requirements_txt_dev_deps)) + self.assertEqual(len(package_json_dev_deps),1) + # devDependencies of package.json file: "@babel/core": ">0.2.0" + self.assertEqual(package_json_dev_deps[0]["component"], "@babel/core") + + def test_dep_scope_production(self): + """ + Run a dependency scan of the current directory, then parse those results + """ + # with open('scanoss-com.pem', 'rb') as f: + # root_certs = f.read() + if MyTestCase.TEST_LOCAL: + server_type = "local" + grpc_client = ScanossGrpc(debug=True, url='localhost:50051') + else: + server_type = "remote" + grpc_client = ScanossGrpc(debug=True) + sc_deps = ScancodeDeps(debug=True) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope=SCOPE.PRODUCTION)) + deps = threaded_deps.responses + files = deps.get("files") + package_json_deps = files[0]["dependencies"] + requirements_txt_deps = files[1].get("dependencies", []) + print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') + print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') + + self.assertNotEquals(len(requirements_txt_deps),5) + self.assertEqual(len(package_json_deps),2) + + self.assertEqual(package_json_deps[0]["component"], "uuid") + self.assertEqual(package_json_deps[1]["component"], "xml-js") + + def test_dep_scope_include(self): + """ + Run a dependency scan of the current directory, then parse those results + """ + # with open('scanoss-com.pem', 'rb') as f: + # root_certs = f.read() + if MyTestCase.TEST_LOCAL: + server_type = "local" + grpc_client = ScanossGrpc(debug=True, url='localhost:50051') + else: + server_type = "remote" + grpc_client = ScanossGrpc(debug=True) + sc_deps = ScancodeDeps(debug=True) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope_include='dependencies')) + deps = threaded_deps.responses + files = deps.get("files") + package_json_deps = files[0]["dependencies"] + requirements_txt_deps = files[1].get("dependencies", []) + print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') + print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') + + # requirements.txt dependencies should be empty due to the filter 'dependencies' + self.assertEqual(len(requirements_txt_deps), 0) + self.assertEqual(len(package_json_deps),2) + # Prod dependencies package.json file: "uuid" and "xml-js" + self.assertEqual(package_json_deps[0]["component"], "uuid") + self.assertEqual(package_json_deps[1]["component"], "xml-js") + + def test_dep_scope_exclude(self): + """ + Run a dependency scan of the current directory, then parse those results + """ + # with open('scanoss-com.pem', 'rb') as f: + # root_certs = f.read() + if MyTestCase.TEST_LOCAL: + server_type = "local" + grpc_client = ScanossGrpc(debug=True, url='localhost:50051') + else: + server_type = "remote" + grpc_client = ScanossGrpc(debug=True) + sc_deps = ScancodeDeps(debug=True) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope_exclude='dependencies,install')) + deps = threaded_deps.responses + files = deps.get("files") + package_json_deps = files[0]["dependencies"] + requirements_txt_deps = files[1].get("dependencies", []) + print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') + print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') + self.assertEqual(len(requirements_txt_deps), 0) + + ## Only dev dependencies should be presents because 'dependencies' and 'install' scopes are excluded + self.assertEqual(len(package_json_deps), 1) + + # Prod dependencies package.json file: "uuid" and "xml-js" + self.assertEqual(package_json_deps[0]["component"], "@babel/core") + + def test_dep_scope_override(self): + """ + Run a dependency scan of the current directory, then parse those results + """ + # with open('scanoss-com.pem', 'rb') as f: + # root_certs = f.read() + if MyTestCase.TEST_LOCAL: + server_type = "local" + grpc_client = ScanossGrpc(debug=True, url='localhost:50051') + else: + server_type = "remote" + grpc_client = ScanossGrpc(debug=True) + sc_deps = ScancodeDeps(debug=True) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope=SCOPE.PRODUCTION ,dep_scope_exclude='dependencies,install')) + deps = threaded_deps.responses + files = deps.get("files") + package_json_deps = files[0]["dependencies"] + requirements_txt_deps = files[1].get("dependencies", []) + print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') + print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') + self.assertEqual(len(requirements_txt_deps), 0) + + ## Only dev dependencies should be presents because 'dependencies' and 'install' scopes are excluded + self.assertEqual(len(package_json_deps), 1) + + # Prod dependencies package.json file: "uuid" and "xml-js" + self.assertEqual(package_json_deps[0]["component"], "@babel/core") + + def test_dependency_scan(self): + """ + Run a dependency scan of the current directory. Dependencies should be returned without scopes + """ + temp_dir = tempfile.gettempdir() + file_name = "dependency-result-output.json" + output_file = os.path.join(temp_dir, file_name) + sc_deps = ScancodeDeps(debug=True, trace=True) + + success = sc_deps.get_dependencies(what_to_scan=".",result_output=output_file) + self.assertTrue(success) + with open(output_file, 'r') as result: + # Parse the JSON data from the file + dependencies = json.load(result) + files = dependencies.get("files") + for file in files: + purls = file.get("purls") + contains_scope = any('scope' in purl for purl in purls) + self.assertFalse(contains_scope) + + os.remove(output_file) if __name__ == '__main__': unittest.main() From 3b7aaf7ab86cd43d664e2446de7fb32f33510bb8 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Sat, 5 Oct 2024 02:34:48 -0300 Subject: [PATCH 170/489] added metadata field to cyclonedx output --- src/scanoss/cyclonedx.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 75df57ac..94b93561 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -25,6 +25,9 @@ import os.path import sys import uuid +import datetime + +from .__init__ import __version__ from .scanossbase import ScanossBase from .spdxlite import SpdxLite @@ -186,6 +189,16 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'specVersion': '1.4', 'serialNumber': f'urn:uuid:{uuid.uuid4()}', 'version': 1, + 'metadata': { + 'timestamp': datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + 'tools': [ + { + 'vendor': 'SCANOSS', + 'name': 'scanoss-py', + 'version': __version__, + } + ] + }, 'components': [], 'vulnerabilities': [] } From 515cb9f5529e871bd001f23e6472ee2a470bf6d3 Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Tue, 8 Oct 2024 11:34:55 -0300 Subject: [PATCH 171/489] update changelog and version --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 337454bd..2cbdf598 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.16.0] - 2024-10-08 +### Added +- Added the `metadata` field to the output in CycloneDX format, now including the fields `timestamp`, `tool vendor`, `tool` and `tool version` + ## [1.15.0] - 2024-09-17 ### Added - Added Results sub-command: @@ -350,3 +354,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.13.0]: https://github.com/scanoss/scanoss.py/compare/v1.12.3...v1.13.0 [1.14.0]: https://github.com/scanoss/scanoss.py/compare/v1.13.0...v1.14.0 [1.15.0]: https://github.com/scanoss/scanoss.py/compare/v1.14.0...v1.15.0 +[1.16.0]: https://github.com/scanoss/scanoss.py/compare/v1.15.0...v1.16.0 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index d10a705a..28ea87a5 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.15.0" +__version__ = "1.16.0" diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 94b93561..91212707 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -27,7 +27,7 @@ import uuid import datetime -from .__init__ import __version__ +from . import __version__ from .scanossbase import ScanossBase from .spdxlite import SpdxLite From 78f12b032fa0956869c549278fca8bbf76ca0233 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Wed, 23 Oct 2024 07:08:01 -0300 Subject: [PATCH 172/489] feat:policy inspect * feat:SP-1647 Adds inspection package * feat:SP-1658 Adds copyleft sub command to define custom list of copyleft licenses * feat: SP-1654 Add undeclared component policy * chore:SP-1662 Adds undeclared component sub command * chore:SP-1663 Adds integration test for copyleft inspect policy * chore:SP-1665 Inspect policies documentation * chore:SP-1668 Adds undeclared component policy integration tests * chore:SP-1669 Handles policies errors * chore:SP-1670 Handles inspect sub command status * chore:SP-1671 Add inspect command documentation in CLIENT_HELP.md * chore:SP-1672 Upgrades CHANGELOG.md file * chore:SP-1676 Refactor on scanoss-py policy analyzer * inspection cleanup * add todo for utcnow call --------- Co-authored-by: eeisegn --- CHANGELOG.md | 9 +- CLIENT_HELP.md | 56 +++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 109 +++++- src/scanoss/inspection/__init__.py | 23 ++ src/scanoss/inspection/copyleft.py | 156 ++++++++ src/scanoss/inspection/policy_check.py | 341 ++++++++++++++++++ .../inspection/undeclared_component.py | 167 +++++++++ src/scanoss/inspection/utils/license_utils.py | 115 ++++++ src/scanoss/results.py | 4 +- src/scanoss/scanossbase.py | 10 + src/scanoss/spdxlite.py | 2 +- tests/policy-inspect-test.py | 221 ++++++++++++ 13 files changed, 1196 insertions(+), 19 deletions(-) create mode 100644 src/scanoss/inspection/__init__.py create mode 100644 src/scanoss/inspection/copyleft.py create mode 100644 src/scanoss/inspection/policy_check.py create mode 100644 src/scanoss/inspection/undeclared_component.py create mode 100644 src/scanoss/inspection/utils/license_utils.py create mode 100644 tests/policy-inspect-test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cbdf598..e43c4c50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.17.0] - 2024-10-17 +### Added +- Added inspect subcommand +- Inspect for copyleft licenses (`scanoss-py inspect copyleft -i scanoss-results.json`) +- Inspect for undeclared components (`scanoss-py inspect undeclared -i scanoss-results.json`) + ## [1.16.0] - 2024-10-08 ### Added - Added the `metadata` field to the output in CycloneDX format, now including the fields `timestamp`, `tool vendor`, `tool` and `tool version` @@ -354,4 +360,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.13.0]: https://github.com/scanoss/scanoss.py/compare/v1.12.3...v1.13.0 [1.14.0]: https://github.com/scanoss/scanoss.py/compare/v1.13.0...v1.14.0 [1.15.0]: https://github.com/scanoss/scanoss.py/compare/v1.14.0...v1.15.0 -[1.16.0]: https://github.com/scanoss/scanoss.py/compare/v1.15.0...v1.16.0 \ No newline at end of file +[1.16.0]: https://github.com/scanoss/scanoss.py/compare/v1.15.0...v1.16.0 +[1.17.0]: https://github.com/scanoss/scanoss.py/compare/v1.16.0...v1.17.0 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index b5cde885..a92212ac 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -139,6 +139,7 @@ scanoss-py scan --help scanoss-py comp scanoss-py comp vulns --help scanoss-py utils +scanoss-py inspect ``` ### Fingerprint a project folder @@ -318,4 +319,59 @@ scanoss-py results results.json --status pending --match-type file You can provide a comma separated list of statuses or match types: ```bash scanoss-py results results.json --status pending,identified --match-type file,snippet +``` + + +### Inspect Commands +The `inspect` command has a suite of sub-commands designed to inspect the results.json. +Details, such as license compliance or component declarations, can be examined. + +For example: +* Copyleft (`copylefet`) +* Undeclared Components (`undeclared`) + +For the latest list of sub-commands, please run: +```bash +scanoss-py insp --help +``` +#### Inspect Copyleft +The following command can be used to inspect for copyleft licenses. +If no output or status flag is defined, details are exposed via stdout and the summary is provided via stderr. +Default format 'json' +```bash +scanoss-py insp copyleft -i scan-results.json +``` + +#### Inspect for copyleft licenses and save results +The following command can be used to inspect for copyleft licenses and save the results. +Default output format 'json'. +```bash +scanoss-py insp copyleft -i scan-results.json --status status.md --output copyleft.json +``` + +#### Inspect for copyleft licenses and save results in Markdown format +The following command can be used to inspect for copyleft licenses and save the results in Markdown format. +```bash +scanoss-py insp copyleft -i scan-results.json --status status.md --output copyleft.md --format md +``` + +#### Inspect for undeclared components +The following command can be used to inspect for undeclared components. +If no output or status flag is defined, details are exposed via stdout and the summary is provided via stderr. +Default output format 'json'. +```bash +scanoss-py insp undeclared -i scan-results.json +``` + +#### Inspect for undeclared components and save results +The following command can be used to inspect for undeclared components and save the results. +Default output format 'json'. +```bash +scanoss-py insp undeclared -i scan-results.json --status undeclared-status.md --output undeclared.json +``` + +#### Inspect for undeclared components and save results in Markdown format +The following command can be used to inspect for undeclared components and save the results in Markdown format. +```bash +scanoss-py insp undeclared -i scan-results.json --status undeclared-status.md --output undeclared.json --format md ``` \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 28ea87a5..a1d58e20 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.16.0" +__version__ = "1.17.0" diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 8bc73475..16a3d8f0 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -25,15 +25,13 @@ import os from pathlib import Path import sys -from array import array - import pypac - +from scanoss.inspection.copyleft import Copyleft +from scanoss.inspection.undeclared_component import UndeclaredComponent from .threadeddependencies import SCOPE -from .scanner import Scanner from .scanoss_settings import ScanossSettings from .scancodedeps import ScancodeDeps -from .scanner import FAST_WINNOWING, Scanner +from .scanner import Scanner from .scantype import ScanType from .filecount import FileCount from .cyclonedx import CycloneDx @@ -171,19 +169,19 @@ def setup_args() -> None: # Component Sub-command: component crypto c_crypto = comp_sub.add_parser('crypto', aliases=['cr'], description=f'Show Cryptographic algorithms: {__version__}', - help='Retreive cryptographic algorithms for the given components') + help='Retrieve cryptographic algorithms for the given components') c_crypto.set_defaults(func=comp_crypto) # Component Sub-command: component vulns c_vulns = comp_sub.add_parser('vulns', aliases=['vulnerabilities', 'vu'], description=f'Show Vulnerability details: {__version__}', - help='Retreive vulnerabilities for the given components') + help='Retrieve vulnerabilities for the given components') c_vulns.set_defaults(func=comp_vulns) # Component Sub-command: component semgrep c_semgrep = comp_sub.add_parser('semgrep', aliases=['sp'], description=f'Show Semgrep findings: {__version__}', - help='Retreive semgrep issues/findings for the given components') + help='Retrieve semgrep issues/findings for the given components') c_semgrep.set_defaults(func=comp_semgrep) # Component Sub-command: component search @@ -299,6 +297,31 @@ def setup_args() -> None: ) p_results.set_defaults(func=results) + + # Sub-command: inspect + p_inspect = subparsers.add_parser('inspect', aliases=['insp', 'ins'], + description=f'Inspect results: {__version__}', + help='Inspect results') + # Sub-parser: inspect + p_inspect_sub = p_inspect.add_subparsers(title='Inspect Commands', dest='subparsercmd', + description='Inspect sub-commands', help='Inspect sub-commands') + # Inspect Sub-command: inspect copyleft + p_copyleft = p_inspect_sub.add_parser('copyleft', aliases=['cp'],description="Inspect for copyleft licenses", help='Inspect for copyleft licenses') + p_copyleft.add_argument('--include', help='List of Copyleft licenses to append to the default list. Provide licenses as a comma-separated list.') + p_copyleft.add_argument('--exclude', help='List of Copyleft licenses to remove from default list. Provide licenses as a comma-separated list.') + p_copyleft.add_argument('--explicit', help='Explicit list of Copyleft licenses to consider. Provide licenses as a comma-separated list.s') + p_copyleft.set_defaults(func=inspect_copyleft) + + # Inspect Sub-command: inspect undeclared + p_undeclared = p_inspect_sub.add_parser('undeclared', aliases=['un'],description="Inspect for undeclared components", help='Inspect for undeclared components') + p_undeclared.set_defaults(func=inspect_undeclared) + + for p in [p_copyleft, p_undeclared]: + p.add_argument('-i', '--input', nargs='?', help='Path to results file') + p.add_argument('-f', '--format',required=False ,choices=['json', 'md'], default='json', help='Output format (default: json)') + p.add_argument('-o', '--output', type=str, help='Save details into a file') + p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file') + # Global Scan command options for p in [p_scan]: p.add_argument('--apiurl', type=str, @@ -344,7 +367,7 @@ def setup_args() -> None: # Help/Trace command options for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto, c_vulns, c_search, - c_versions, c_semgrep, p_results]: + c_versions, c_semgrep, p_results, p_undeclared, p_copyleft]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -357,9 +380,10 @@ def setup_args() -> None: parser.print_help() # No sub command subcommand, print general help exit(1) else: - if (args.subparser == 'utils' or args.subparser == 'ut' or - args.subparser == 'component' or args.subparser == 'comp') \ - and not args.subparsercmd: + if ((args.subparser == 'utils' or args.subparser == 'ut' or + args.subparser == 'component' or args.subparser == 'comp' or + args.subparser == 'inspect' or args.subparser == 'insp' or args.subparser == 'ins') + and not args.subparsercmd): parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed exit(1) args.func(parser, args) # Execute the function associated with the sub-command @@ -778,6 +802,64 @@ def convert(parser, args): if not success: exit(1) +def inspect_copyleft(parser, args): + """ + Run the "inspect" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if args.input is None: + print_stderr('Please specify an input file to inspect') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + output: str = None + if args.output: + output = args.output + open(output, 'w').close() + + status_output: str = None + if args.status: + status_output = args.status + open(status_output, 'w').close() + + i_copyleft = Copyleft(debug=args.debug, trace=args.trace, quiet=args.quiet, filepath=args.input, + format_type=args.format, status=status_output, output=output, include=args.include, + exclude=args.exclude, explicit=args.explicit) + status, _ = i_copyleft.run() + sys.exit(status) + +def inspect_undeclared(parser, args): + """ + Run the "inspect" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if args.input is None: + print_stderr('Please specify an input file to inspect') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + output: str = None + if args.output: + output = args.output + open(output, 'w').close() + + status_output: str = None + if args.status: + status_output = args.status + open(status_output, 'w').close() + i_undeclared = UndeclaredComponent(debug=args.debug, trace=args.trace, quiet=args.quiet, + filepath=args.input, format_type=args.format, + status=status_output, output=output) + status, _ = i_undeclared.run() + sys.exit(status) def utils_certloc(*_): """ @@ -787,7 +869,6 @@ def utils_certloc(*_): import certifi print(f'CA Cert File: {certifi.where()}') - def utils_cert_download(_, args): """ Run the "utils cert-download" sub-command @@ -820,7 +901,7 @@ def utils_cert_download(_, args): else: cn = cert_components.get('CN') if not args.quiet: - print_stderr(f'Centificate {index} - CN: {cn}') + print_stderr(f'Certificate {index} - CN: {cn}') if sys.version_info[0] >= 3: print((crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file) # Print the downloaded PEM certificate else: diff --git a/src/scanoss/inspection/__init__.py b/src/scanoss/inspection/__init__.py new file mode 100644 index 00000000..5243c416 --- /dev/null +++ b/src/scanoss/inspection/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py new file mode 100644 index 00000000..808388f8 --- /dev/null +++ b/src/scanoss/inspection/copyleft.py @@ -0,0 +1,156 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +from typing import Dict, Any +from scanoss.inspection.policy_check import PolicyCheck, PolicyStatus + +class Copyleft(PolicyCheck): + """ + SCANOSS Copyleft class + Inspects components for copyleft licenses + """ + + def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, + format_type: str = 'json', status: str = None, output: str = None, include: str = None, + exclude: str = None, explicit: str = None): + """ + Initialize the Copyleft class. + + :param debug: Enable debug mode + :param trace: Enable trace mode (default True) + :param quiet: Enable quiet mode + :param filepath: Path to the file containing component data + :param format_type: Output format ('json' or 'md') + :param status: Path to save the status output + :param output: Path to save detailed output + :param include: Licenses to include in the analysis + :param exclude: Licenses to exclude from the analysis + :param explicit: Explicitly defined licenses + """ + super().__init__(debug, trace, quiet, filepath, format_type, status, output, name='Copyleft Policy') + self.license_util.init(include, exclude, explicit) + self.filepath = filepath + self.format = format + self.output = output + self.status = status + self.include = include + self.exclude = exclude + self.explicit = explicit + + def _json(self, components: list) -> Dict[str, Any]: + """ + Format the components with copyleft licenses as JSON. + + :param components: List of components with copyleft licenses + :return: Dictionary with formatted JSON details and summary + """ + details = {} + if len(components) > 0: + details = { 'components': components } + return { + 'details': json.dumps(details, indent=2), + 'summary': f'{len(components)} component(s) with copyleft licenses were found.' + } + + def _markdown(self, components: list) -> Dict[str,Any]: + """ + Format the components with copyleft licenses as Markdown. + + :param components: List of components with copyleft licenses + :return: Dictionary with formatted Markdown details and summary + """ + headers = ['Component', 'Version', 'License', 'URL', 'Copyleft'] + centered_columns = [1, 4] + rows: [[]]= [] + for component in components: + for lic in component['licenses']: + row = [ + component['purl'], + component['version'], + lic['spdxid'], + lic['url'], + 'YES' if lic['copyleft'] else 'NO' + ] + rows.append(row) + # End license loop + # End component loop + return { + 'details': f'### Copyleft licenses\n{self.generate_table(headers,rows,centered_columns)}', + 'summary' : f'{len(components)} component(s) with copyleft licenses were found.' + } + + def _filter_components_with_copyleft_licenses(self, components: list) -> list: + """ + Filter the components list to include only those with copyleft licenses. + + :param components: List of all components + :return: List of components with copyleft licenses + """ + filtered_components = [] + for component in components: + copyleft_licenses = [lic for lic in component['licenses'] if lic['copyleft']] + if copyleft_licenses: + filtered_component = component + filtered_component['licenses'] = copyleft_licenses + del filtered_component['status'] + filtered_components.append(filtered_component) + # End component loop + self.print_debug(f'Copyleft components: {filtered_components}') + return filtered_components + + def run(self): + """ + Run the copyleft license inspection process. + + This method performs the following steps: + 1. Get all components + 2. Filter components with copyleft licenses + 3. Format the results + 4. Save the output to files if required + + :return: Dictionary containing the inspection results + """ + self._debug() + # Get the components from the results + components = self._get_components() + if components is None: + return PolicyStatus.ERROR.value, {} + # Get a list of copyleft components if they exist + copyleft_components = self._filter_components_with_copyleft_licenses(components) + # Get a formatter for the output results + formatter = self._get_formatter() + if formatter is None: + return PolicyStatus.ERROR.value, {} + # Format the results + results = formatter(copyleft_components) + ## Save outputs if required + self.print_to_file_or_stdout(results['details'], self.output) + self.print_to_file_or_stderr(results['summary'], self.status) + # Check to see if we have policy violations + if len(copyleft_components) <= 0: + return PolicyStatus.FAIL.value, results + return PolicyStatus.SUCCESS.value, results +# +# End of Copyleft Class +# diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py new file mode 100644 index 00000000..19dc7c1f --- /dev/null +++ b/src/scanoss/inspection/policy_check.py @@ -0,0 +1,341 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +import os.path +from abc import abstractmethod +from enum import Enum +from typing import Callable, List, Dict, Any +from scanoss.inspection.utils.license_utils import LicenseUtil +from scanoss.scanossbase import ScanossBase + +class PolicyStatus(Enum): + """ + Enumeration representing the status of a policy check. + + Attributes: + SUCCESS (int): Indicates that the policy check passed successfully (value: 0). + FAIL (int): Indicates that the policy check failed (value: 1). + ERROR (int): Indicates that an error occurred during the policy check (value: 2). + """ + SUCCESS = 0 + FAIL = 1 + ERROR = 2 +# +# End of PolicyStatus Class +# + +class ComponentID(Enum): + """ + Enumeration representing different types of software components. + + Attributes: + FILE (str): Represents a file component (value: "file"). + SNIPPET (str): Represents a code snippet component (value: "snippet"). + DEPENDENCY (str): Represents a dependency component (value: "dependency"). + """ + FILE = "file" + SNIPPET = "snippet" + DEPENDENCY = "dependency" +# +# End of ComponentID Class +# + +class PolicyCheck(ScanossBase): + """ + A base class for implementing various software policy checks. + + This class provides a framework for policy checking, including methods for + processing components, generating output in different formats. + + Attributes: + VALID_FORMATS (set): A set of valid output formats ('md', 'json'). + + Inherits from: + ScanossBase: A base class providing common functionality for SCANOSS-related operations. + """ + + VALID_FORMATS = {'md', 'json'} + + def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, + format_type: str = None, status: str = None, output: str = None, name: str = None): + super().__init__(debug, trace, quiet) + self.license_util = LicenseUtil() + self.filepath = filepath + self.name = name + self.output = output + self.format_type = format_type + self.status = status + self.results = self._load_input_file() + + @abstractmethod + def run(self): + """ + Execute the policy check process. + + This abstract method should be implemented by subclasses to perform specific + policy checks. The general structure of this method typically includes: + 1. Retrieving components + 2. Filtering components based on specific criteria + 3. Formatting the results + 4. Saving the output to files if required + + :return: A tuple containing: + - First element: PolicyStatus enum value (SUCCESS, FAIL, or ERROR) + - Second element: Dictionary containing the inspection results + """ + pass + + @abstractmethod + def _json(self, components: list) -> Dict[str, Any]: + """ + Format the policy checks results as JSON. + This method should be implemented by subclasses to create a Markdown representation + of the policy check results. + + :param components: List of components to be formatted. + :return: A dictionary containing two keys: + - 'details': A JSON-formatted string with the full list of components + - 'summary': A string summarizing the number of components found + """ + pass + + @abstractmethod + def _markdown(self, components: list) -> Dict[str, Any]: + """ + Generate Markdown output for the policy check results. + + This method should be implemented by subclasses to create a Markdown representation + of the policy check results. + + :param components: List of components to be included in the output. + :return: A dictionary representing the Markdown output. + """ + pass + + def _append_component(self,components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: + """ + Append a new component to the component's dictionary. + + This function creates a new entry in the components dictionary for the given component, + or updates an existing entry if the component already exists. It also processes the + licenses associated with the component. + + :param components: The existing dictionary of components + :param new_component: The new component to be added or updated + :return: The updated components dictionary + """ + component_key = f"{new_component['purl'][0]}@{new_component['version']}" + components[component_key] = { + 'purl': new_component['purl'][0], + 'version': new_component['version'], + 'licenses': {}, + 'status': new_component['status'], + } + if not new_component.get('licenses'): + self.print_stderr(f'WARNING: Results missing licenses. Skipping.') + return components + # Process licenses for this component + for l in new_component['licenses']: + if l.get('name'): + spdxid = l['name'] + components[component_key]['licenses'][spdxid] = { + 'spdxid': spdxid, + 'copyleft': self.license_util.is_copyleft(spdxid), + 'url': self.license_util.get_spdx_url(spdxid), + } + return components + + def _get_components_from_results(self,results: Dict[str, Any]) -> list or None: + """ + Process the results dictionary to extract and format component information. + + This function iterates through the results dictionary, identifying components from + different sources (files, snippets, and dependencies). It consolidates this information + into a list of unique components, each with its associated licenses and other details. + + :param results: A dictionary containing the raw results of a component scan + :return: A list of dictionaries, each representing a unique component with its details + """ + if results is None: + self.print_stderr(f'ERROR: Results cannot be empty') + return None + components = {} + for component in results.values(): + for c in component: + component_id = c.get('id') + if not component_id: + self.print_stderr(f'WARNING: Result missing id. Skipping.') + continue + if component_id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: + if not c.get('purl'): + self.print_stderr(f'WARNING: Result missing purl. Skipping.') + continue + if len(c.get('purl')) <= 0: + self.print_stderr(f'WARNING: Result missing purls. Skipping.') + continue + if not c.get('version'): + self.print_stderr(f'WARNING: Result missing version. Skipping.') + continue + component_key = f"{c['purl'][0]}@{c['version']}" + # Initialize or update the component entry + if component_key not in components: + components = self._append_component(components, c) + if c['id'] == ComponentID.DEPENDENCY.value: + if c.get('dependency') is None: + continue + for d in c['dependencies']: + if not d.get('purl'): + self.print_stderr(f'WARNING: Result missing purl. Skipping.') + continue + if len(d.get('purl')) <= 0: + self.print_stderr(f'WARNING: Result missing purls. Skipping.') + continue + if not d.get('version'): + self.print_stderr(f'WARNING: Result missing version. Skipping.') + continue + component_key = f"{d['purl'][0]}@{d['version']}" + if component_key not in components: + components = self._append_component(components, d) + # End of dependencies loop + # End if + # End of component loop + # End of results loop + results = list(components.values()) + for component in results: + component['licenses'] = list(component['licenses'].values()) + + return results + + def generate_table(self, headers, rows, centered_columns=None): + """ + Generate a Markdown table. + + :param headers: List of headers for the table. + :param rows: List of rows for the table. + :param centered_columns: List of column indices to be centered. + :return: A string representing the Markdown table. + """ + col_sep = ' | ' + centered_column_set = set(centered_columns or []) + if headers is None: + self.print_stderr('ERROR: Header are no set') + return None + # Decide which separator to use + def create_separator(index): + if centered_columns is None: + return '-' + return ':-:' if index in centered_column_set else '-' + # Build the row separator + row_separator = col_sep + col_sep.join(create_separator(index) for index, _ in enumerate(headers)) + col_sep + # build table rows + table_rows = [col_sep + col_sep.join(headers) + col_sep, row_separator] + table_rows.extend(col_sep + col_sep.join(row) + col_sep for row in rows) + return '\n'.join(table_rows) + + def _get_formatter(self)-> Callable[[List[dict]], Dict[str,Any]] or None: + """ + Get the appropriate formatter function based on the specified format. + + :return: Formatter function (either _json or _markdown) + """ + valid_format = self._is_valid_format() + if not valid_format: + return None + # a map of which format function to return + function_map = { + 'json': self._json, + 'md': self._markdown + } + return function_map[self.format_type] + + def _debug(self): + """ + Print debug information about the policy check. + + This method prints various attributes of the PolicyCheck instance for debugging purposes. + """ + if self.debug: + self.print_stderr(f'Policy: {self.name}') + self.print_stderr(f'Format: {self.format_type}') + self.print_stderr(f'Status: {self.status}') + self.print_stderr(f'Output: {self.output}') + self.print_stderr(f'Input: {self.filepath}') + + def _is_valid_format(self) -> bool: + """ + Validate if the format specified is supported. + + This method checks if the format stored in format is one of the + valid formats defined in self.VALID_FORMATS. + + :return: bool: True if the format is valid, False otherwise. + """ + if self.format_type not in self.VALID_FORMATS: + valid_formats_str = ', '.join(self.VALID_FORMATS) + self.print_stderr(f'ERROR: Invalid format "{self.format_type}". Valid formats are: {valid_formats_str}') + return False + return True + + def _load_input_file(self): + """ + Load the result.json file + + Returns: + Dict[str, Any]: The parsed JSON data + """ + if not os.path.exists(self.filepath): + self.print_stderr(f'ERROR: The file "{self.filepath}" does not exist.') + return None + with open(self.filepath, "r") as jsonfile: + try: + return json.load(jsonfile) + except Exception as e: + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + return None + + def _get_components(self): + """ + Retrieve and process components from the preloaded results. + + This method performs the following steps: + 1. Checks if the results have been previously loaded (self.results). + 2. Extracts and processes components from the loaded results. + + :return: A list of processed components, or None if an error occurred during any step. + Possible reasons for returning None include: + - Results not loaded (self.results is None) + - Failure to extract components from the results + + Note: + - This method assumes that the results have been previously loaded and stored in self.results. + - If results is None, the method returns None without performing any further operations. + - The actual processing of components is delegated to the _get_components_from_results method. + """ + if self.results is None: + return None + components = self._get_components_from_results(self.results) + return components +# +# End of PolicyCheck Class +# diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py new file mode 100644 index 00000000..ce593cf8 --- /dev/null +++ b/src/scanoss/inspection/undeclared_component.py @@ -0,0 +1,167 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +from typing import Dict, Any +from scanoss.inspection.policy_check import PolicyCheck, PolicyStatus + +class UndeclaredComponent(PolicyCheck): + """ + SCANOSS UndeclaredComponent class + Inspects for undeclared components + """ + + def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, + format_type: str = 'json', status: str = None, output: str = None): + """ + Initialize the UndeclaredComponent class. + + :param debug: Enable debug mode + :param trace: Enable trace mode (default True) + :param quiet: Enable quiet mode + :param filepath: Path to the file containing component data + :param format_type: Output format ('json' or 'md') + :param status: Path to save status output + :param output: Path to save detailed output + """ + super().__init__(debug, trace, quiet, filepath, format_type, status, output, + name='Undeclared Components Policy') + self.filepath = filepath + self.format = format + self.output = output + self.status = status + + def _get_undeclared_component(self, components: list)-> list or None: + """ + Filter the components list to include only undeclared components. + + :param components: List of all components + :return: List of undeclared components + """ + if components is None: + self.print_stderr(f'WARNING: No components provided!') + return None + undeclared_components = [] + for component in components: + if component['status'] == 'pending': + del component['status'] + undeclared_components.append(component) + # end component loop + return undeclared_components + + def _get_summary(self, components: list) -> str: + """ + Get a summary of the undeclared components. + + :param components: List of all components + :return: Component summary markdown + """ + summary = f'{len(components)} undeclared component(s) were found.\n' + if len(components) > 0: + summary += (f' Add the following snippet into your `sbom.json` file \n' + f' ```json \n {json.dumps(self._generate_sbom_file(components), indent=2)} ``` \n ') + return summary + + def _json(self, components: list) -> Dict[str, Any]: + """ + Format the undeclared components as JSON. + + :param components: List of undeclared components + :return: Dictionary with formatted JSON details and summary + """ + details = {} + if len(components) > 0: + details = {'components': components} + return { + 'details': json.dumps(details, indent=2), + 'summary': self._get_summary(components), + } + + def _markdown(self, components: list) -> Dict[str,Any]: + """ + Format the undeclared components as Markdown. + + :param components: List of undeclared components + :return: Dictionary with formatted Markdown details and summary + """ + headers = ['Component', 'Version', 'License'] + rows: [[]]= [] + # TODO look at using SpdxLite license name lookup method + for component in components: + licenses = " - ".join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) + rows.append([component['purl'], component['version'], licenses]) + return { + 'details': f'### Undeclared components\n{self.generate_table(headers,rows)}', + 'summary': self._get_summary(components), + } + + def _generate_sbom_file(self, components: list) -> list: + """ + Generate a list of PURLs for the SBOM file. + + :param components: List of undeclared components + :return: List of dictionaries containing PURLs + """ + sbom = {} + if components is None: + self.print_stderr(f'WARNING: No components provided!') + else: + for component in components: + sbom[component['purl']] = { 'purl': component['purl'] } + return list(sbom.values()) + + def run(self): + """ + Run the undeclared component inspection process. + + This method performs the following steps: + 1. Get all components + 2. Filter undeclared components + 3. Format the results + 4. Save the output to files if required + + :return: Dictionary containing the inspection results + """ + self._debug() + components = self._get_components() + if components is None: + return PolicyStatus.ERROR.value, {} + # Get undeclared component summary (if any) + undeclared_components = self._get_undeclared_component(components) + if undeclared_components is None: + return PolicyStatus.ERROR.value, {} + self.print_debug(f'Undeclared components: {undeclared_components}') + formatter = self._get_formatter() + if formatter is None: + return PolicyStatus.ERROR.value, {} + results = formatter(undeclared_components) + # Output the results + self.print_to_file_or_stdout(results['details'], self.output) + self.print_to_file_or_stderr(results['summary'], self.status) + # Determine if the filter found results or not + if len(undeclared_components) <= 0: + return PolicyStatus.FAIL.value, results + return PolicyStatus.SUCCESS.value, results +# +# End of UndeclaredComponent Class +# diff --git a/src/scanoss/inspection/utils/license_utils.py b/src/scanoss/inspection/utils/license_utils.py new file mode 100644 index 00000000..9111d346 --- /dev/null +++ b/src/scanoss/inspection/utils/license_utils.py @@ -0,0 +1,115 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +from scanoss.scanossbase import ScanossBase + +DEFAULT_COPYLEFT_LICENSES = { + 'agpl-3.0-only', 'artistic-1.0', 'artistic-2.0', 'cc-by-sa-4.0', 'cddl-1.0', 'cddl-1.1', 'cecill-2.1', + 'epl-1.0', 'epl-2.0', 'gfdl-1.1-only', 'gfdl-1.2-only', 'gfdl-1.3-only', 'gpl-1.0-only', 'gpl-2.0-only', + 'gpl-3.0-only', 'lgpl-2.1-only', 'lgpl-3.0-only', 'mpl-1.1', 'mpl-2.0', 'sleepycat', 'watcom-1.0' +} + +class LicenseUtil(ScanossBase): + """ + A utility class for handling software licenses, particularly copyleft licenses. + + This class provides functionality to initialize, manage, and query a set of + copyleft licenses. It also offers a method to generate URLs for license information. + """ + BASE_SPDX_ORG_URL = 'https://spdx.org/licenses' + BASE_OSADL_URL = 'https://www.osadl.org/fileadmin/checklists/unreflicenses' + + def __init__(self,debug: bool = False, trace: bool = True, quiet: bool = False): + super().__init__(debug, trace, quiet) + self.default_copyleft_licenses = set(DEFAULT_COPYLEFT_LICENSES) + self.copyleft_licenses = set() + + def init(self, include: str = None, exclude: str = None, explicit: str = None): + """ + Initialize the set of copyleft licenses based on user input. + + This method allows for customization of the copyleft license set by: + - Setting an explicit list of licenses + - Including additional licenses to the default set + - Excluding specific licenses from the default set + + :param include: Comma-separated string of licenses to include + :param exclude: Comma-separated string of licenses to exclude + :param explicit: Comma-separated string of licenses to use exclusively + """ + if self.debug: + self.print_stderr(f'Include Copyleft licenses: ${include}') + self.print_stderr(f'Exclude Copyleft licenses: ${exclude}') + self.print_stderr(f'Explicit Copyleft licenses: ${explicit}') + if explicit: + explicit = explicit.strip() + if explicit: + exp = [item.strip().lower() for item in explicit.split(',')] + self.copyleft_licenses = set(exp) + self.print_debug(f'Copyleft licenses: ${self.copyleft_licenses}') + return + # If no explicit licenses were set, set default ones + self.copyleft_licenses = self.default_copyleft_licenses.copy() + if include: + include = include.strip() + if include: + inc =[item.strip().lower() for item in include.split(',')] + self.copyleft_licenses.update(inc) + if exclude: + exclude = exclude.strip() + if exclude: + inc = [item.strip().lower() for item in exclude.split(',')] + for lic in inc: + self.copyleft_licenses.discard(lic) + self.print_debug(f'Copyleft licenses: ${self.copyleft_licenses}') + + def is_copyleft(self, spdxid: str) -> bool: + """ + Check if a given license is considered copyleft. + + :param spdxid: The SPDX identifier of the license to check + :return: True if the license is copyleft, False otherwise + """ + return spdxid.lower() in self.copyleft_licenses + + def get_spdx_url(self, spdxid: str) -> str: + """ + Generate the URL for the SPDX page of a license. + + :param spdxid: The SPDX identifier of the license + :return: The URL of the SPDX page for the given license + """ + return f'{self.BASE_SPDX_ORG_URL}/{spdxid}.html' + + + def get_osadl_url(self, spdxid: str) -> str: + """ + Generate the URL for the OSADL (Open Source Automation Development Lab) page of a license. + + :param spdxid: The SPDX identifier of the license + :return: The URL of the OSADL page for the given license + """ + return f'{self.BASE_OSADL_URL}/{spdxid}.txt' +# +# End of LicenseUtil Class +# diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 7174f53a..438386d1 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -86,7 +86,7 @@ def __init__( self.output_file = output_file self.output_format = output_format - def _load_file(self, file: str) -> Dict[str, Any]: + def load_file(self, file: str) -> Dict[str, Any]: """Load the JSON file Args: @@ -106,7 +106,7 @@ def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: Load the file and transform the data into a list of dictionaries with the filename and the file data """ - raw_data = self._load_file(file) + raw_data = self.load_file(file) return self._transform_data(raw_data) @staticmethod diff --git a/src/scanoss/scanossbase.py b/src/scanoss/scanossbase.py index 1694bfe3..974e2bf2 100644 --- a/src/scanoss/scanossbase.py +++ b/src/scanoss/scanossbase.py @@ -89,3 +89,13 @@ def print_to_file_or_stdout(self, msg: str, file: str = None): f.write(msg) else: self.print_stdout(msg) + + def print_to_file_or_stderr(self, msg: str, file: str = None): + """ + Print message to file if provided or stderr + """ + if file: + with open(file, "w") as f: + f.write(msg) + else: + self.print_stderr(msg) diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 94c9abc7..98a55c86 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -175,7 +175,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: # pip3 install jsonschema # jsonschema -i spdxlite.json <(curl https://raw.githubusercontent.com/spdx/spdx-spec/v2.2/schemas/spdx-schema.json) # Validation can also be done online here: https://tools.spdx.org/app/validate/ - now = datetime.datetime.utcnow() + now = datetime.datetime.utcnow() # TODO replace with recommended format md5hex = hashlib.md5(f'{raw_data}-{now}'.encode('utf-8')).hexdigest() data = { 'spdxVersion': 'SPDX-2.2', diff --git a/tests/policy-inspect-test.py b/tests/policy-inspect-test.py new file mode 100644 index 00000000..62fb3427 --- /dev/null +++ b/tests/policy-inspect-test.py @@ -0,0 +1,221 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +import os +import re +import unittest + +from scanoss.inspection.copyleft import Copyleft +from scanoss.inspection.undeclared_component import UndeclaredComponent + + +class MyTestCase(unittest.TestCase): + + + """ + Inspect for copyleft licenses + """ + def test_copyleft_policy(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir,'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json') + copyleft.run() + self.assertEqual(True, True) + + """ + Inspect for copyleft licenses empty path + """ + def test_copyleft_policy_empty_path(self): + copyleft = Copyleft(filepath='', format_type='json') + success, results = copyleft.run() + self.assertTrue(success,2) + + + """ + Inspect for empty copyleft licenses + """ + def test_empty_copyleft_policy(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result-no-copyleft.json" + input_file_name = os.path.join(script_dir,'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json') + status,results = copyleft.run() + details = json.loads(results['details']) + self.assertEqual(status, 1) + self.assertEqual(details, {}) + self.assertEqual(results['summary'], '0 component(s) with copyleft licenses were found.') + + """ + Inspect for copyleft licenses include + """ + def test_copyleft_policy_include(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json', include='MIT') + status, results = copyleft.run() + has_mit_license = False + details = json.loads(results['details']) + for component in details['components']: + for license in component['licenses']: + if license['spdxid'] == 'MIT': + has_mit_license = True + break + + self.assertEqual(status,0) + self.assertEqual(has_mit_license, True) + + """ + Inspect for copyleft licenses exclude + """ + def test_copyleft_policy_exclude(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json', exclude='GPL-2.0-only') + status,results = copyleft.run() + details = json.loads(results['details']) + self.assertEqual(details, {}) + self.assertEqual(status, 1) + + """ + Inspect for copyleft licenses explicit + """ + def test_copyleft_policy_explicit(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='MIT') + status, results = copyleft.run() + details = json.loads(results['details']) + self.assertEqual(len(details['components']), 1) + self.assertEqual(status,0) + + """ + Inspect for copyleft licenses empty explicit licenses (should set the default ones) + """ + def test_copyleft_policy_empty_explicit(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='') + status, results = copyleft.run() + details = json.loads(results['details']) + self.assertEqual(len(details['components']), 5) + self.assertEqual(status,0) + + + """ + Export copyleft licenses in Markdown + """ + def test_copyleft_policy_markdown(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='md', explicit='MIT') + status, results = copyleft.run() + expected_detail_output = ('### Copyleft licenses \n | Component | Version | License | URL | Copyleft |\n' + ' | - | :-: | - | - | :-: |\n' + ' | pkg:github/scanoss/engine | 4.0.4 | MIT | https://spdx.org/licenses/MIT.html | YES | ') + expected_summary_output = '1 component(s) with copyleft licenses were found.' + self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', results['details']), + re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output)) + self.assertEqual(results['summary'], expected_summary_output) + self.assertEqual(status, 0) + + ## Undeclared Components Policy Tests ## + + """ + Inspect for undeclared components empty path + """ + def test_copyleft_policy_empty_path(self): + copyleft = UndeclaredComponent(filepath='', format_type='json') + success, results = copyleft.run() + self.assertTrue(success,2) + + + """ + Inspect for undeclared components + """ + def test_undeclared_policy(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir,'data', file_name) + undeclared = UndeclaredComponent(filepath=input_file_name, format_type='json') + status, results = undeclared.run() + details = json.loads(results['details']) + summary = results['summary'] + expected_summary_output = """3 undeclared component(s) were found. + Add the following snippet into your `sbom.json` file + ```json + [ + { + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + } + ]``` + """ + self.assertEqual(len(details['components']), 3) + self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', + '', expected_summary_output)) + self.assertEqual(status, 0) + + """ + Undeclared component markdown output + """ + def test_undeclared_policy_markdown(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md') + status, results = undeclared.run() + details = results['details'] + summary = results['summary'] + expected_details_output = """ ### Undeclared components + | Component | Version | License | + | - | - | - | + | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | + | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | + | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | """ + + expected_summary_output = """3 undeclared component(s) were found. + Add the following snippet into your `sbom.json` file + ```json + [ + { + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + } + ]``` + """ + self.assertEqual(status, 0) + self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', + '', expected_details_output)) + self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), + re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) \ No newline at end of file From 876bfce1537ba72f569735233397b229dcd95f7a Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz <166400360+ortizjeronimo@users.noreply.github.com> Date: Wed, 23 Oct 2024 08:56:26 -0300 Subject: [PATCH 173/489] bug: Corrected SPDX date format * removes miliseconds from the timestamp field in SDPX output --- src/scanoss/spdxlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 98a55c86..7ca92a3e 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -183,7 +183,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'SPDXID': f'SPDXRef-{md5hex}', 'name': 'SCANOSS-SBOM', 'creationInfo': { - 'created': now.strftime('%Y-%m-%dT%H:%M:%S') + now.strftime('.%f')[:4] + 'Z', + 'created': now.strftime('%Y-%m-%dT%H:%M:%SZ'), 'creators': [f'Tool: SCANOSS-PY: {__version__}', f'Person: {getpass.getuser()}'] }, 'documentNamespace': f'https://spdx.org/spdxdocs/scanoss-py-{__version__}-{md5hex}', From cd496e8ef84f802b503956403eac4c2ed101a9a3 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:25:36 -0300 Subject: [PATCH 174/489] chore:SP-1689 Adds SPDX date formatting to CHANGELOG.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e43c4c50..09cf9685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.17.0] - 2024-10-17 +## [1.17.0] - 2024-10-23 ### Added - Added inspect subcommand - Inspect for copyleft licenses (`scanoss-py inspect copyleft -i scanoss-results.json`) - Inspect for undeclared components (`scanoss-py inspect undeclared -i scanoss-results.json`) +### Fixed +- Fixed SPDX date format ## [1.16.0] - 2024-10-08 ### Added From 02af5de87d40873bdd04ad31b3b52007c531132f Mon Sep 17 00:00:00 2001 From: agusgroh Date: Mon, 14 Oct 2024 12:23:07 -0300 Subject: [PATCH 175/489] feat:SP-1647 Adds inspection package --- src/scanoss/inspections/__init__.py | 23 +++++++ src/scanoss/inspections/copyleft.py | 28 +++++++++ src/scanoss/inspections/policyCheck.py | 21 +++++++ .../inspections/undeclaredComponent.py | 7 +++ .../inspections/utils/license_utils.py | 27 +++++++++ src/scanoss/inspections/utils/result_utils.py | 60 +++++++++++++++++++ 6 files changed, 166 insertions(+) create mode 100644 src/scanoss/inspections/__init__.py create mode 100644 src/scanoss/inspections/copyleft.py create mode 100644 src/scanoss/inspections/policyCheck.py create mode 100644 src/scanoss/inspections/undeclaredComponent.py create mode 100644 src/scanoss/inspections/utils/license_utils.py create mode 100644 src/scanoss/inspections/utils/result_utils.py diff --git a/src/scanoss/inspections/__init__.py b/src/scanoss/inspections/__init__.py new file mode 100644 index 00000000..d2ed05b0 --- /dev/null +++ b/src/scanoss/inspections/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/inspections/copyleft.py b/src/scanoss/inspections/copyleft.py new file mode 100644 index 00000000..01baba5e --- /dev/null +++ b/src/scanoss/inspections/copyleft.py @@ -0,0 +1,28 @@ +from scanoss.inspections.policyCheck import PolicyCheck +from scanoss.inspections.utils.result_utils import get_components + + +class Copyleft(PolicyCheck): + + def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False, filepath: str = None, + format: str = None, status: str = None): + super().__init__(debug, trace, quiet,filepath,format,status) + + + def run(self): + print("File path",self.file_path) + results = self.result.load_file(self.file_path) + ##print("Results",results) + components = get_components(results) + print(components) + + """ + Inspect for copyleft licenses + """ + + + """ + self.format... + Call function depending on the format output + """ + diff --git a/src/scanoss/inspections/policyCheck.py b/src/scanoss/inspections/policyCheck.py new file mode 100644 index 00000000..aa5c53b0 --- /dev/null +++ b/src/scanoss/inspections/policyCheck.py @@ -0,0 +1,21 @@ +from abc import abstractmethod + +from scanoss.results import Results +from scanoss.scanossbase import ScanossBase + + +class PolicyCheck(ScanossBase): + + result: Results + + + def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False, filepath: str = None, format: str = None, status: str = None): + super().__init__(debug, trace, quiet) + self.file_path = filepath + self.format = format + self.status = status + self.result = Results(debug, trace, quiet, filepath) + + @abstractmethod + def run(self): + pass \ No newline at end of file diff --git a/src/scanoss/inspections/undeclaredComponent.py b/src/scanoss/inspections/undeclaredComponent.py new file mode 100644 index 00000000..b577168c --- /dev/null +++ b/src/scanoss/inspections/undeclaredComponent.py @@ -0,0 +1,7 @@ +from scanoss.inspections.policyCheck import PolicyCheck + + +class UndeclaredComponent(PolicyCheck): + + def run(self): + pass \ No newline at end of file diff --git a/src/scanoss/inspections/utils/license_utils.py b/src/scanoss/inspections/utils/license_utils.py new file mode 100644 index 00000000..26e8378b --- /dev/null +++ b/src/scanoss/inspections/utils/license_utils.py @@ -0,0 +1,27 @@ +import os +import logging + +class LicenseUtil: + BASE_OSADL_URL = 'https://spdx.org/licenses' + HTML = 'html' + + def __init__(self): + self.default_copyleft_licenses = set([ + 'gpl-1.0-only', 'gpl-2.0-only', 'gpl-3.0-only', 'agpl-3.0-only', + 'sleepycat', 'watcom-1.0', 'gfdl-1.1-only', 'gfdl-1.2-only', + 'gfdl-1.3-only', 'lgpl-2.1-only', 'lgpl-3.0-only', 'mpl-1.1', + 'mpl-2.0', 'epl-1.0', 'epl-2.0', 'cddl-1.0', 'cddl-1.1', + 'cecill-2.1', 'artistic-1.0', 'artistic-2.0', 'cc-by-sa-4.0' + ]) + self.copyleft_licenses = set() + self.copyleft_licenses = self.default_copyleft_licenses.copy() + + + def is_copyleft(self, spdxid: str) -> bool: + return spdxid.lower() in self.copyleft_licenses + + def get_osadl(self, spdxid: str) -> str: + return f"{self.BASE_OSADL_URL}/{spdxid}.{self.HTML}" + + +license_util = LicenseUtil() \ No newline at end of file diff --git a/src/scanoss/inspections/utils/result_utils.py b/src/scanoss/inspections/utils/result_utils.py new file mode 100644 index 00000000..80f34d39 --- /dev/null +++ b/src/scanoss/inspections/utils/result_utils.py @@ -0,0 +1,60 @@ +from collections import defaultdict +from enum import Enum +from typing import Dict, Any, List + +from scanoss.inspections.utils.license_utils import license_util + + +class ComponentID(Enum): + FILE = "file" + SNIPPET = "snippet" + DEPENDENCY = "dependency" + +def _append_component(components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: + + component_key = f"{new_component['purl'][0]}@{new_component['version']}" + components[component_key] = { + 'purl': new_component['purl'][0], + 'version': new_component['version'], + 'licenses': {} + } + + # Process licenses for this component + for l in new_component['licenses']: + spdxid = l['name'] + components[component_key]['licenses'][spdxid] = { + 'spdxid': spdxid, + 'copyleft': license_util.is_copyleft(spdxid), + 'url': l.get('url'), + 'count': 1 + } + + return components + + +def get_components(results: Dict[str, Any]) -> []: + components = {} + for component in results.values(): + for c in component: + if c['id'] in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: + component_key = f"{c['purl'][0]}@{c['version']}" + + # Initialize or update the component entry + if component_key not in components: + components = _append_component(components, c) + + if c['id'] == ComponentID.DEPENDENCY.value: + for d in c['dependencies']: + component_key = f"{d['purl'][0]}@{d['version']}" + + if component_key not in components: + components = _append_component(components, d) + + + + + results = list(components.values()) + for component in results: + component['licenses'] = list(component['licenses'].values()) + + return results From d7ba60a854da92d919ee381df51211b75424858c Mon Sep 17 00:00:00 2001 From: agusgroh Date: Tue, 15 Oct 2024 13:07:49 -0300 Subject: [PATCH 176/489] feat:SP-1658 Adds copyleft sub command to define custom list of copyleft licenses --- .../inspection/utils/markdown_utils.py | 23 +++++++++++++++ .../utils/result_utils.py | 18 ++++++------ src/scanoss/inspections/__init__.py | 23 --------------- src/scanoss/inspections/copyleft.py | 28 ------------------- src/scanoss/inspections/policyCheck.py | 21 -------------- .../inspections/undeclaredComponent.py | 7 ----- .../inspections/utils/license_utils.py | 27 ------------------ 7 files changed, 32 insertions(+), 115 deletions(-) create mode 100644 src/scanoss/inspection/utils/markdown_utils.py rename src/scanoss/{inspections => inspection}/utils/result_utils.py (84%) delete mode 100644 src/scanoss/inspections/__init__.py delete mode 100644 src/scanoss/inspections/copyleft.py delete mode 100644 src/scanoss/inspections/policyCheck.py delete mode 100644 src/scanoss/inspections/undeclaredComponent.py delete mode 100644 src/scanoss/inspections/utils/license_utils.py diff --git a/src/scanoss/inspection/utils/markdown_utils.py b/src/scanoss/inspection/utils/markdown_utils.py new file mode 100644 index 00000000..e4b9e902 --- /dev/null +++ b/src/scanoss/inspection/utils/markdown_utils.py @@ -0,0 +1,23 @@ +def generate_table(headers, rows, centered_columns=None): + """ + Generate Markdown table + :param headers: List of headers + :param rows: Rows + :param centered_columns: List with centered columns + """ + COL_SEP = ' | ' + centered_column_set = set(centered_columns or []) + def create_separator(header, index): + if centered_columns is None: + return '-' + return ':-:' if index in centered_column_set else '-' + + row_separator = COL_SEP + COL_SEP.join( + create_separator(header, index) for index, header in enumerate(headers) + ) + COL_SEP + + table_rows = [COL_SEP + COL_SEP.join(headers) + COL_SEP] + table_rows.append(row_separator) + table_rows.extend(COL_SEP + COL_SEP.join(row) + COL_SEP for row in rows) + + return '\n'.join(table_rows) \ No newline at end of file diff --git a/src/scanoss/inspections/utils/result_utils.py b/src/scanoss/inspection/utils/result_utils.py similarity index 84% rename from src/scanoss/inspections/utils/result_utils.py rename to src/scanoss/inspection/utils/result_utils.py index 80f34d39..e6d6083d 100644 --- a/src/scanoss/inspections/utils/result_utils.py +++ b/src/scanoss/inspection/utils/result_utils.py @@ -1,8 +1,9 @@ from collections import defaultdict +from dataclasses import dataclass from enum import Enum -from typing import Dict, Any, List +from typing import Dict, Any, List, Optional -from scanoss.inspections.utils.license_utils import license_util +from scanoss.inspection.utils.license_utils import license_util class ComponentID(Enum): @@ -10,6 +11,7 @@ class ComponentID(Enum): SNIPPET = "snippet" DEPENDENCY = "dependency" + def _append_component(components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: component_key = f"{new_component['purl'][0]}@{new_component['version']}" @@ -25,14 +27,13 @@ def _append_component(components: Dict[str, Any], new_component: Dict[str, Any]) components[component_key]['licenses'][spdxid] = { 'spdxid': spdxid, 'copyleft': license_util.is_copyleft(spdxid), - 'url': l.get('url'), - 'count': 1 + 'url': l.get('url') } return components -def get_components(results: Dict[str, Any]) -> []: +def get_components(results: Dict[str, Any]) -> list: components = {} for component in results.values(): for c in component: @@ -49,10 +50,9 @@ def get_components(results: Dict[str, Any]) -> []: if component_key not in components: components = _append_component(components, d) - - - - + # End of for loop + # End if + # End if results = list(components.values()) for component in results: component['licenses'] = list(component['licenses'].values()) diff --git a/src/scanoss/inspections/__init__.py b/src/scanoss/inspections/__init__.py deleted file mode 100644 index d2ed05b0..00000000 --- a/src/scanoss/inspections/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" - SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -""" diff --git a/src/scanoss/inspections/copyleft.py b/src/scanoss/inspections/copyleft.py deleted file mode 100644 index 01baba5e..00000000 --- a/src/scanoss/inspections/copyleft.py +++ /dev/null @@ -1,28 +0,0 @@ -from scanoss.inspections.policyCheck import PolicyCheck -from scanoss.inspections.utils.result_utils import get_components - - -class Copyleft(PolicyCheck): - - def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False, filepath: str = None, - format: str = None, status: str = None): - super().__init__(debug, trace, quiet,filepath,format,status) - - - def run(self): - print("File path",self.file_path) - results = self.result.load_file(self.file_path) - ##print("Results",results) - components = get_components(results) - print(components) - - """ - Inspect for copyleft licenses - """ - - - """ - self.format... - Call function depending on the format output - """ - diff --git a/src/scanoss/inspections/policyCheck.py b/src/scanoss/inspections/policyCheck.py deleted file mode 100644 index aa5c53b0..00000000 --- a/src/scanoss/inspections/policyCheck.py +++ /dev/null @@ -1,21 +0,0 @@ -from abc import abstractmethod - -from scanoss.results import Results -from scanoss.scanossbase import ScanossBase - - -class PolicyCheck(ScanossBase): - - result: Results - - - def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False, filepath: str = None, format: str = None, status: str = None): - super().__init__(debug, trace, quiet) - self.file_path = filepath - self.format = format - self.status = status - self.result = Results(debug, trace, quiet, filepath) - - @abstractmethod - def run(self): - pass \ No newline at end of file diff --git a/src/scanoss/inspections/undeclaredComponent.py b/src/scanoss/inspections/undeclaredComponent.py deleted file mode 100644 index b577168c..00000000 --- a/src/scanoss/inspections/undeclaredComponent.py +++ /dev/null @@ -1,7 +0,0 @@ -from scanoss.inspections.policyCheck import PolicyCheck - - -class UndeclaredComponent(PolicyCheck): - - def run(self): - pass \ No newline at end of file diff --git a/src/scanoss/inspections/utils/license_utils.py b/src/scanoss/inspections/utils/license_utils.py deleted file mode 100644 index 26e8378b..00000000 --- a/src/scanoss/inspections/utils/license_utils.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import logging - -class LicenseUtil: - BASE_OSADL_URL = 'https://spdx.org/licenses' - HTML = 'html' - - def __init__(self): - self.default_copyleft_licenses = set([ - 'gpl-1.0-only', 'gpl-2.0-only', 'gpl-3.0-only', 'agpl-3.0-only', - 'sleepycat', 'watcom-1.0', 'gfdl-1.1-only', 'gfdl-1.2-only', - 'gfdl-1.3-only', 'lgpl-2.1-only', 'lgpl-3.0-only', 'mpl-1.1', - 'mpl-2.0', 'epl-1.0', 'epl-2.0', 'cddl-1.0', 'cddl-1.1', - 'cecill-2.1', 'artistic-1.0', 'artistic-2.0', 'cc-by-sa-4.0' - ]) - self.copyleft_licenses = set() - self.copyleft_licenses = self.default_copyleft_licenses.copy() - - - def is_copyleft(self, spdxid: str) -> bool: - return spdxid.lower() in self.copyleft_licenses - - def get_osadl(self, spdxid: str) -> str: - return f"{self.BASE_OSADL_URL}/{spdxid}.{self.HTML}" - - -license_util = LicenseUtil() \ No newline at end of file From 02ba28849c7e8f5e5e699bec4d55a0fb29ac9c88 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Wed, 16 Oct 2024 07:40:48 -0300 Subject: [PATCH 177/489] feat: SP-1654 Add undeclared component policy --- src/scanoss/inspection/utils/result_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scanoss/inspection/utils/result_utils.py b/src/scanoss/inspection/utils/result_utils.py index e6d6083d..1e25056d 100644 --- a/src/scanoss/inspection/utils/result_utils.py +++ b/src/scanoss/inspection/utils/result_utils.py @@ -18,7 +18,8 @@ def _append_component(components: Dict[str, Any], new_component: Dict[str, Any]) components[component_key] = { 'purl': new_component['purl'][0], 'version': new_component['version'], - 'licenses': {} + 'licenses': {}, + 'status': new_component['status'], } # Process licenses for this component From 98f11259b478c4d675e2555fffd761ba7fe7b325 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Wed, 16 Oct 2024 11:02:34 -0300 Subject: [PATCH 178/489] chore:SP-1663 Adds integration test for copyleft inspect policy --- src/scanoss/inspection/utils/result_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/scanoss/inspection/utils/result_utils.py b/src/scanoss/inspection/utils/result_utils.py index 1e25056d..a8924760 100644 --- a/src/scanoss/inspection/utils/result_utils.py +++ b/src/scanoss/inspection/utils/result_utils.py @@ -1,7 +1,5 @@ -from collections import defaultdict -from dataclasses import dataclass from enum import Enum -from typing import Dict, Any, List, Optional +from typing import Dict, Any from scanoss.inspection.utils.license_utils import license_util From 6e9b75e1314c353963fc36559a042baf8519cb70 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Wed, 16 Oct 2024 12:02:20 -0300 Subject: [PATCH 179/489] chore:SP-1665 Inspect policies documentation --- src/scanoss/inspection/utils/result_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/scanoss/inspection/utils/result_utils.py b/src/scanoss/inspection/utils/result_utils.py index a8924760..4d089924 100644 --- a/src/scanoss/inspection/utils/result_utils.py +++ b/src/scanoss/inspection/utils/result_utils.py @@ -11,7 +11,17 @@ class ComponentID(Enum): def _append_component(components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: + """ + Append a new component to the components dictionary. + This function creates a new entry in the components dictionary for the given component, + or updates an existing entry if the component already exists. It also processes the + licenses associated with the component. + + :param components: The existing dictionary of components + :param new_component: The new component to be added or updated + :return: The updated components dictionary + """ component_key = f"{new_component['purl'][0]}@{new_component['version']}" components[component_key] = { 'purl': new_component['purl'][0], @@ -33,6 +43,16 @@ def _append_component(components: Dict[str, Any], new_component: Dict[str, Any]) def get_components(results: Dict[str, Any]) -> list: + """ + Process the results dictionary to extract and format component information. + + This function iterates through the results dictionary, identifying components from + different sources (files, snippets, and dependencies). It consolidates this information + into a list of unique components, each with its associated licenses and other details. + + :param results: A dictionary containing the raw results of a component scan + :return: A list of dictionaries, each representing a unique component with its details + """ components = {} for component in results.values(): for c in component: From 4391efd274191affe8e55cd2138a4dc0d7b1a28c Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Thu, 24 Oct 2024 12:33:15 -0300 Subject: [PATCH 180/489] bug:SP-1696 Fixes policy summary output --- CHANGELOG.md | 7 +- src/scanoss/__init__.py | 2 +- src/scanoss/inspection/copyleft.py | 8 +- .../inspection/undeclared_component.py | 8 +- tests/data/result-no-copyleft.json | 238 ++++++++++ tests/data/result.json | 419 ++++++++++++++++++ tests/policy-inspect-test.py | 4 +- 7 files changed, 674 insertions(+), 12 deletions(-) create mode 100644 tests/data/result-no-copyleft.json create mode 100644 tests/data/result.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 09cf9685..155d666f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.17.1] - 2024-10-24 +### Fixed +- Fixed policy summary output + ## [1.17.0] - 2024-10-23 ### Added - Added inspect subcommand @@ -363,4 +367,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.14.0]: https://github.com/scanoss/scanoss.py/compare/v1.13.0...v1.14.0 [1.15.0]: https://github.com/scanoss/scanoss.py/compare/v1.14.0...v1.15.0 [1.16.0]: https://github.com/scanoss/scanoss.py/compare/v1.15.0...v1.16.0 -[1.17.0]: https://github.com/scanoss/scanoss.py/compare/v1.16.0...v1.17.0 \ No newline at end of file +[1.17.0]: https://github.com/scanoss/scanoss.py/compare/v1.16.0...v1.17.0 +[1.17.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.0...v1.17.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index a1d58e20..a4c015b4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.17.0" +__version__ = "1.17.1" diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index 808388f8..c99621d5 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -69,8 +69,8 @@ def _json(self, components: list) -> Dict[str, Any]: if len(components) > 0: details = { 'components': components } return { - 'details': json.dumps(details, indent=2), - 'summary': f'{len(components)} component(s) with copyleft licenses were found.' + 'details': f'{json.dumps(details, indent=2)}\n', + 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n' } def _markdown(self, components: list) -> Dict[str,Any]: @@ -96,8 +96,8 @@ def _markdown(self, components: list) -> Dict[str,Any]: # End license loop # End component loop return { - 'details': f'### Copyleft licenses\n{self.generate_table(headers,rows,centered_columns)}', - 'summary' : f'{len(components)} component(s) with copyleft licenses were found.' + 'details': f'### Copyleft licenses\n{self.generate_table(headers,rows,centered_columns)}\n', + 'summary' : f'{len(components)} component(s) with copyleft licenses were found.\n' } def _filter_components_with_copyleft_licenses(self, components: list) -> list: diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index ce593cf8..f4f6ce22 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -78,8 +78,8 @@ def _get_summary(self, components: list) -> str: """ summary = f'{len(components)} undeclared component(s) were found.\n' if len(components) > 0: - summary += (f' Add the following snippet into your `sbom.json` file \n' - f' ```json \n {json.dumps(self._generate_sbom_file(components), indent=2)} ``` \n ') + summary += (f'Add the following snippet into your `sbom.json` file\n' + f'\n```json\n{json.dumps(self._generate_sbom_file(components), indent=2)}\n```\n') return summary def _json(self, components: list) -> Dict[str, Any]: @@ -93,7 +93,7 @@ def _json(self, components: list) -> Dict[str, Any]: if len(components) > 0: details = {'components': components} return { - 'details': json.dumps(details, indent=2), + 'details': f'{json.dumps(details, indent=2)}\n', 'summary': self._get_summary(components), } @@ -111,7 +111,7 @@ def _markdown(self, components: list) -> Dict[str,Any]: licenses = " - ".join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) rows.append([component['purl'], component['version'], licenses]) return { - 'details': f'### Undeclared components\n{self.generate_table(headers,rows)}', + 'details': f'### Undeclared components\n{self.generate_table(headers,rows)}\n', 'summary': self._get_summary(components), } diff --git a/tests/data/result-no-copyleft.json b/tests/data/result-no-copyleft.json new file mode 100644 index 00000000..8695018b --- /dev/null +++ b/tests/data/result-no-copyleft.json @@ -0,0 +1,238 @@ +{ + "inc/crc32c.h": [ + { + "id": "none", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + } + } + ], + "inc/json.h": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/inc/json.h", + "file_hash": "e91a03b850651dd56dd979ba92668a19", + "file_url": "https://api.osskb.org/file_contents/e91a03b850651dd56dd979ba92668a19", + "id": "file", + "latest": "1.3.4", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "e91a03b850651dd56dd979ba92668a19", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "inc/log.h": [ + { + "component": "scanner.c", + "file": "scanner.c-1.1.4/external/inc/log.h", + "file_hash": "36ae4f65e9302539357a64ddb8e35c28", + "file_url": "https://api.osskb.org/file_contents/36ae4f65e9302539357a64ddb8e35c28", + "id": "file", + "latest": "1.3.4", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-01-11", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "36ae4f65e9302539357a64ddb8e35c28", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "1dcd883dc73d16f3ce2cf4f1f9854c7b", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.1.4" + } + ], + "inc/winnowing.h": [ + { + "component": "engine", + "file": "external/inc/winnowing.h", + "file_hash": "d58b03e8ef411204db0fb991f778444a", + "file_url": "https://api.osskb.org/file_contents/d58b03e8ef411204db0fb991f778444a", + "id": "file", + "latest": "5.4.8", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/engine" + ], + "release_date": "2024-02-27", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "d58b03e8ef411204db0fb991f778444a", + "status": "identified", + "url": "https://github.com/scanoss/engine", + "url_hash": "5107cd431b8b6c7836c84e997bab01ec", + "url_stats": {}, + "vendor": "scanoss", + "version": "5.4.0" + } + ], + "src/crc32c.c": [ + { + "component": "wfp", + "file": "wfp-6afc1f6163d1d6c8d03ff5211a0571118e08da1f/src/external/crc32c/crc32c.c", + "file_hash": "0fe279946d388ef07d9c3f6e3ffb8ebe", + "file_url": "https://api.osskb.org/file_contents/0fe279946d388ef07d9c3f6e3ffb8ebe", + "id": "file", + "latest": "0ed473d", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/wfp" + ], + "release_date": "2020-07-12", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "0fe279946d388ef07d9c3f6e3ffb8ebe", + "status": "pending", + "url": "https://github.com/scanoss/wfp", + "url_hash": "9b36f30d422d7f77854f298f63c55256", + "url_stats": {}, + "vendor": "scanoss", + "version": "6afc1f6" + } + ], + "src/json.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/src/json.c", + "file_hash": "8e4d433c1547b59681379e9fe9960546", + "file_url": "https://api.osskb.org/file_contents/8e4d433c1547b59681379e9fe9960546", + "id": "file", + "latest": "1.3.4", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "8e4d433c1547b59681379e9fe9960546", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "src/log.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/src/log.c", + "file_hash": "f00c8a010806ff1593b15c7cbff7e594", + "file_url": "https://api.osskb.org/file_contents/f00c8a010806ff1593b15c7cbff7e594", + "id": "file", + "latest": "1.3.4", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "f00c8a010806ff1593b15c7cbff7e594", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "src/winnowing.c": [ + { + "component": "engine", + "file": "external/src/winnowing.c", + "file_hash": "b298d620599a6ad04e2613dfdc9d3f7b", + "file_url": "https://api.osskb.org/file_contents/b298d620599a6ad04e2613dfdc9d3f7b", + "id": "snippet", + "latest": "4.3.4", + "licenses": [], + "lines": "32-161", + "matched": "80%", + "oss_lines": "29-158", + "purl": [ + "pkg:github/scanoss/engine" + ], + "release_date": "2020-12-30", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "af368f0dfee072472cf57f0694975b28", + "status": "pending", + "url": "https://github.com/scanoss/engine", + "url_hash": "0c0c7aaf65ab9e2e10562d188ff3e511", + "url_stats": {}, + "vendor": "scanoss", + "version": "4.0.4" + } + ] +} diff --git a/tests/data/result.json b/tests/data/result.json new file mode 100644 index 00000000..9602102e --- /dev/null +++ b/tests/data/result.json @@ -0,0 +1,419 @@ +{ + "inc/crc32c.h": [ + { + "id": "none", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + } + } + ], + "inc/json.h": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/inc/json.h", + "file_hash": "e91a03b850651dd56dd979ba92668a19", + "file_url": "https://api.osskb.org/file_contents/e91a03b850651dd56dd979ba92668a19", + "id": "file", + "latest": "1.3.4", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/BSD-2-Clause.txt", + "copyleft": "no", + "name": "BSD-2-Clause", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "no", + "source": "file_header", + "url": "https://spdx.org/licenses/BSD-2-Clause.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/BSD-2-Clause.txt", + "copyleft": "no", + "name": "BSD-2-Clause", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "no", + "source": "scancode", + "url": "https://spdx.org/licenses/BSD-2-Clause.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "e91a03b850651dd56dd979ba92668a19", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "inc/log.h": [ + { + "component": "scanner.c", + "file": "scanner.c-1.1.4/external/inc/log.h", + "file_hash": "36ae4f65e9302539357a64ddb8e35c28", + "file_url": "https://api.osskb.org/file_contents/36ae4f65e9302539357a64ddb8e35c28", + "id": "file", + "latest": "1.3.4", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-01-11", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "36ae4f65e9302539357a64ddb8e35c28", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "1dcd883dc73d16f3ce2cf4f1f9854c7b", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.1.4" + } + ], + "inc/winnowing.h": [ + { + "component": "engine", + "file": "external/inc/winnowing.h", + "file_hash": "d58b03e8ef411204db0fb991f778444a", + "file_url": "https://api.osskb.org/file_contents/d58b03e8ef411204db0fb991f778444a", + "id": "file", + "latest": "5.4.8", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "license_file", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/engine" + ], + "release_date": "2024-02-27", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "d58b03e8ef411204db0fb991f778444a", + "status": "identified", + "url": "https://github.com/scanoss/engine", + "url_hash": "5107cd431b8b6c7836c84e997bab01ec", + "url_stats": {}, + "vendor": "scanoss", + "version": "5.4.0" + } + ], + "src/crc32c.c": [ + { + "component": "wfp", + "file": "wfp-6afc1f6163d1d6c8d03ff5211a0571118e08da1f/src/external/crc32c/crc32c.c", + "file_hash": "0fe279946d388ef07d9c3f6e3ffb8ebe", + "file_url": "https://api.osskb.org/file_contents/0fe279946d388ef07d9c3f6e3ffb8ebe", + "id": "file", + "latest": "0ed473d", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/Zlib.txt", + "copyleft": "no", + "name": "Zlib", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "no", + "source": "scancode", + "url": "https://spdx.org/licenses/Zlib.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/Zlib.txt", + "copyleft": "no", + "name": "Zlib", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "no", + "source": "file_header", + "url": "https://spdx.org/licenses/Zlib.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "license_file", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/wfp" + ], + "release_date": "2020-07-12", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "0fe279946d388ef07d9c3f6e3ffb8ebe", + "status": "pending", + "url": "https://github.com/scanoss/wfp", + "url_hash": "9b36f30d422d7f77854f298f63c55256", + "url_stats": {}, + "vendor": "scanoss", + "version": "6afc1f6" + } + ], + "src/json.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/src/json.c", + "file_hash": "8e4d433c1547b59681379e9fe9960546", + "file_url": "https://api.osskb.org/file_contents/8e4d433c1547b59681379e9fe9960546", + "id": "file", + "latest": "1.3.4", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/BSD-2-Clause.txt", + "copyleft": "no", + "name": "BSD-2-Clause", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "no", + "source": "file_header", + "url": "https://spdx.org/licenses/BSD-2-Clause.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "8e4d433c1547b59681379e9fe9960546", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "src/log.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/src/log.c", + "file_hash": "f00c8a010806ff1593b15c7cbff7e594", + "file_url": "https://api.osskb.org/file_contents/f00c8a010806ff1593b15c7cbff7e594", + "id": "file", + "latest": "1.3.4", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "f00c8a010806ff1593b15c7cbff7e594", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "src/winnowing.c": [ + { + "component": "engine", + "file": "external/src/winnowing.c", + "file_hash": "b298d620599a6ad04e2613dfdc9d3f7b", + "file_url": "https://api.osskb.org/file_contents/b298d620599a6ad04e2613dfdc9d3f7b", + "id": "snippet", + "latest": "4.3.4", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "file_spdx_tag", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-1.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-1.0-or-later", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "no", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-1.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-or-later.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-or-later", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "scancode", + "url": "https://spdx.org/licenses/GPL-2.0-or-later.html" + }, + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt", + "copyleft": "yes", + "incompatible_with": "Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, ECL-2.0, FTL, IJG, Minpack, OpenSSL, PHP-3.01, Python-2.0, zlib-acknowledgement, XFree86-1.1", + "name": "GPL-2.0-only", + "osadl_updated": "2024-08-11T02:21:00+00:00", + "patent_hints": "yes", + "source": "component_declared", + "url": "https://spdx.org/licenses/GPL-2.0-only.html" + }, + { + "name": "MIT", + "patent_hints": "no", + "copyleft": "no", + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "osadl_updated": "2024-09-20T09:32:00+0000", + "source": "scancode", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "32-161", + "matched": "80%", + "oss_lines": "29-158", + "purl": [ + "pkg:github/scanoss/engine" + ], + "release_date": "2020-12-30", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "af368f0dfee072472cf57f0694975b28", + "status": "identified", + "url": "https://github.com/scanoss/engine", + "url_hash": "0c0c7aaf65ab9e2e10562d188ff3e511", + "url_stats": {}, + "vendor": "scanoss", + "version": "4.0.4" + } + ] +} diff --git a/tests/policy-inspect-test.py b/tests/policy-inspect-test.py index 62fb3427..79ad5e88 100644 --- a/tests/policy-inspect-test.py +++ b/tests/policy-inspect-test.py @@ -65,7 +65,7 @@ def test_empty_copyleft_policy(self): details = json.loads(results['details']) self.assertEqual(status, 1) self.assertEqual(details, {}) - self.assertEqual(results['summary'], '0 component(s) with copyleft licenses were found.') + self.assertEqual(results['summary'], '0 component(s) with copyleft licenses were found.\n') """ Inspect for copyleft licenses include @@ -139,7 +139,7 @@ def test_copyleft_policy_markdown(self): expected_detail_output = ('### Copyleft licenses \n | Component | Version | License | URL | Copyleft |\n' ' | - | :-: | - | - | :-: |\n' ' | pkg:github/scanoss/engine | 4.0.4 | MIT | https://spdx.org/licenses/MIT.html | YES | ') - expected_summary_output = '1 component(s) with copyleft licenses were found.' + expected_summary_output = '1 component(s) with copyleft licenses were found.\n' self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', results['details']), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output)) self.assertEqual(results['summary'], expected_summary_output) From 93eddfefafd4b8101936a9d73e33b25a35395bcf Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 29 Oct 2024 08:30:46 -0300 Subject: [PATCH 181/489] bug: SP-1715 Fixes Parsing of Dependency Field in Policy Checks --- CHANGELOG.md | 5 ++++ src/scanoss/__init__.py | 2 +- src/scanoss/inspection/policy_check.py | 36 ++++++++++++++++------- tests/data/result.json | 36 ++++++++++++++++++++++- tests/policy-inspect-test.py | 40 ++++++++++++++++++-------- 5 files changed, 95 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 155d666f..28425a83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.17.2] - 2024-10-29 +### Fixed +- Fixed parsing of dependencies in Policy Checks + ## [1.17.1] - 2024-10-24 ### Fixed - Fixed policy summary output @@ -369,3 +373,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.16.0]: https://github.com/scanoss/scanoss.py/compare/v1.15.0...v1.16.0 [1.17.0]: https://github.com/scanoss/scanoss.py/compare/v1.16.0...v1.17.0 [1.17.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.0...v1.17.1 +[1.17.2]: https://github.com/scanoss/scanoss.py/compare/v1.17.1...v1.17.2 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index a4c015b4..6a2d87d3 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.17.1" +__version__ = "1.17.2" diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index 19dc7c1f..fd9a369b 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -133,7 +133,8 @@ def _markdown(self, components: list) -> Dict[str, Any]: """ pass - def _append_component(self,components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: + def _append_component(self,components: Dict[str, Any], new_component: Dict[str, Any], + id: str, status: str) -> Dict[str, Any]: """ Append a new component to the component's dictionary. @@ -143,15 +144,25 @@ def _append_component(self,components: Dict[str, Any], new_component: Dict[str, :param components: The existing dictionary of components :param new_component: The new component to be added or updated + :param id: The new component ID + :param status: The new component status :return: The updated components dictionary """ - component_key = f"{new_component['purl'][0]}@{new_component['version']}" + + # Determine the component key and purl based on component type + if id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: + purl = new_component['purl'][0] # Take first purl for these component types + else: + purl = new_component['purl'] + + component_key = f"{purl}@{new_component['version']}" components[component_key] = { - 'purl': new_component['purl'][0], - 'version': new_component['version'], - 'licenses': {}, - 'status': new_component['status'], + 'purl': purl, + 'version': new_component['version'], + 'licenses': {}, + 'status': status, } + if not new_component.get('licenses'): self.print_stderr(f'WARNING: Results missing licenses. Skipping.') return components @@ -187,6 +198,10 @@ def _get_components_from_results(self,results: Dict[str, Any]) -> list or None: if not component_id: self.print_stderr(f'WARNING: Result missing id. Skipping.') continue + status = c.get('status') + if not component_id: + self.print_stderr(f'WARNING: Result missing status. Skipping.') + continue if component_id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: if not c.get('purl'): self.print_stderr(f'WARNING: Result missing purl. Skipping.') @@ -200,9 +215,10 @@ def _get_components_from_results(self,results: Dict[str, Any]) -> list or None: component_key = f"{c['purl'][0]}@{c['version']}" # Initialize or update the component entry if component_key not in components: - components = self._append_component(components, c) + components = self._append_component(components, c, component_id, status) + if c['id'] == ComponentID.DEPENDENCY.value: - if c.get('dependency') is None: + if c.get('dependencies') is None: continue for d in c['dependencies']: if not d.get('purl'): @@ -214,9 +230,9 @@ def _get_components_from_results(self,results: Dict[str, Any]) -> list or None: if not d.get('version'): self.print_stderr(f'WARNING: Result missing version. Skipping.') continue - component_key = f"{d['purl'][0]}@{d['version']}" + component_key = f"{d['purl']}@{d['version']}" if component_key not in components: - components = self._append_component(components, d) + components = self._append_component(components, d, component_id, status) # End of dependencies loop # End if # End of component loop diff --git a/tests/data/result.json b/tests/data/result.json index 9602102e..fcf0df98 100644 --- a/tests/data/result.json +++ b/tests/data/result.json @@ -415,5 +415,39 @@ "vendor": "scanoss", "version": "4.0.4" } - ] + ], + "example_codebase/dependencies/package.json": [ + { + "dependencies": [ + { + "component": "@electron/rebuild", + "licenses": [ + { + "is_spdx_approved": true, + "name": "MIT", + "spdx_id": "MIT" + } + ], + "purl": "pkg:npm/%40electron/rebuild", + "url": "https://www.npmjs.com/package/%40electron/rebuild", + "version": "3.7.0" + }, + { + "component": "@emotion/react", + "licenses": [ + { + "is_spdx_approved": true, + "name": "MIT", + "spdx_id": "MIT" + } + ], + "purl": "pkg:npm/%40emotion/react", + "url": "https://www.npmjs.com/package/%40emotion/react", + "version": "11.13.3" + } + ], + "id": "dependency", + "status": "pending" + } + ] } diff --git a/tests/policy-inspect-test.py b/tests/policy-inspect-test.py index 79ad5e88..7497b941 100644 --- a/tests/policy-inspect-test.py +++ b/tests/policy-inspect-test.py @@ -110,7 +110,7 @@ def test_copyleft_policy_explicit(self): copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='MIT') status, results = copyleft.run() details = json.loads(results['details']) - self.assertEqual(len(details['components']), 1) + self.assertEqual(len(details['components']), 3) self.assertEqual(status,0) """ @@ -138,8 +138,10 @@ def test_copyleft_policy_markdown(self): status, results = copyleft.run() expected_detail_output = ('### Copyleft licenses \n | Component | Version | License | URL | Copyleft |\n' ' | - | :-: | - | - | :-: |\n' - ' | pkg:github/scanoss/engine | 4.0.4 | MIT | https://spdx.org/licenses/MIT.html | YES | ') - expected_summary_output = '1 component(s) with copyleft licenses were found.\n' + '| pkg:github/scanoss/engine | 4.0.4 | MIT | https://spdx.org/licenses/MIT.html | YES | \n' + ' | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | https://spdx.org/licenses/MIT.html | YES |\n' + '| pkg:npm/%40emotion/react | 11.13.3 | MIT | https://spdx.org/licenses/MIT.html | YES | \n') + expected_summary_output = '3 component(s) with copyleft licenses were found.\n' self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', results['details']), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output)) self.assertEqual(results['summary'], expected_summary_output) @@ -167,7 +169,7 @@ def test_undeclared_policy(self): status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] - expected_summary_output = """3 undeclared component(s) were found. + expected_summary_output = """5 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json [ @@ -176,10 +178,16 @@ def test_undeclared_policy(self): }, { "purl": "pkg:github/scanoss/wfp" + }, + { + "purl": "pkg:npm/%40electron/rebuild" + }, + { + "purl": "pkg:npm/%40emotion/react" } ]``` """ - self.assertEqual(len(details['components']), 3) + self.assertEqual(len(details['components']), 5) self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) self.assertEqual(status, 0) @@ -200,18 +208,26 @@ def test_undeclared_policy_markdown(self): | - | - | - | | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | - | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | """ + | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | + | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | + | pkg:npm/%40emotion/react | 11.13.3 | MIT | """ - expected_summary_output = """3 undeclared component(s) were found. + expected_summary_output = """5 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json [ { - "purl": "pkg:github/scanoss/scanner.c" - }, - { - "purl": "pkg:github/scanoss/wfp" - } + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + }, + { + "purl": "pkg:npm/%40electron/rebuild" + }, + { + "purl": "pkg:npm/%40emotion/react" + } ]``` """ self.assertEqual(status, 0) From c32110b6928385b30746b56400ee0176bae0483b Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 29 Oct 2024 10:09:47 -0300 Subject: [PATCH 182/489] chore:SP-1718 Adds supplier to spdx packages --- CHANGELOG.md | 2 ++ src/scanoss/spdxlite.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28425a83..6185074e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.17.2] - 2024-10-29 ### Fixed - Fixed parsing of dependencies in Policy Checks +### Added +- Added supplier to SPDX packages ## [1.17.1] - 2024-10-24 ### Fixed diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 7ca92a3e..d72ebc80 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -180,7 +180,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: data = { 'spdxVersion': 'SPDX-2.2', 'dataLicense': 'CC0-1.0', - 'SPDXID': f'SPDXRef-{md5hex}', + 'SPDXID': f'SPDXRef-DOCUMENT', 'name': 'SCANOSS-SBOM', 'creationInfo': { 'created': now.strftime('%Y-%m-%dT%H:%M:%SZ'), @@ -214,6 +214,8 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: comp_name = comp.get('component') comp_ver = comp.get('version') purl_ver = f'{purl}@{comp_ver}' + vendor = comp.get('vendor', 'NOASSERTION') + supplier = f"Organization: {vendor}" if vendor != 'NOASSERTION' else vendor purl_hash = hashlib.md5(f'{purl_ver}'.encode('utf-8')).hexdigest() purl_spdx = f'SPDXRef-{purl_hash}' data['documentDescribes'].append(purl_spdx) @@ -227,6 +229,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'licenseConcluded': 'NOASSERTION', 'filesAnalyzed': False, 'copyrightText': 'NOASSERTION', + 'supplier': supplier, 'externalRefs': [{ 'referenceCategory': 'PACKAGE-MANAGER', 'referenceLocator': purl_ver, From 8d33e2d442c3241d8d9e68ac32d7632552950e82 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 29 Oct 2024 15:01:47 +0100 Subject: [PATCH 183/489] bug: Handle legacy sbom.json in scanoss settings class --- src/scanoss/scanoss_settings.py | 51 ++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index e8aad44e..21230295 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -1,30 +1,30 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the 'Software'), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the 'Software'), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ import json from pathlib import Path -from typing import Dict, List, TypedDict +from typing import List, TypedDict from .scanossbase import ScanossBase @@ -123,7 +123,12 @@ def _get_bom(self): list: If using SBOM.json """ if self.settings_file_type == "legacy": - return self.data.get("components", []) + if isinstance(self.data, list): + return self.data + elif isinstance(self.data, dict) and self.data.get("components"): + return self.data.get("components") + else: + return [] return self.data.get("bom", {}) def get_bom_include(self) -> List[BomEntry]: From adc5cb2f9bfd913041d738d7a66f2673fa9b66ad Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 29 Oct 2024 15:05:32 +0100 Subject: [PATCH 184/489] bug: Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6185074e..dc611dc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.17.2] - 2024-10-29 ### Fixed - Fixed parsing of dependencies in Policy Checks +- Fixed legacy SBOM.json support ### Added - Added supplier to SPDX packages From 8ca43185d47cec8a78e510b5e2202c7dbbd42675 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Fri, 1 Nov 2024 07:46:39 -0300 Subject: [PATCH 185/489] chore:SP-1729 Changes undeclared component summary output --- CHANGELOG.md | 4 +- .../inspection/undeclared_component.py | 16 +++-- tests/policy-inspect-test.py | 62 ++++++++++--------- 3 files changed, 48 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc611dc9..efa79ec5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.17.2] - 2024-10-29 +## [1.17.2] - 2024-11-01 ### Fixed - Fixed parsing of dependencies in Policy Checks - Fixed legacy SBOM.json support ### Added - Added supplier to SPDX packages +### Changed +- Changed undeclared summary output ## [1.17.1] - 2024-10-24 ### Fixed diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index f4f6ce22..d111334f 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -115,20 +115,26 @@ def _markdown(self, components: list) -> Dict[str,Any]: 'summary': self._get_summary(components), } - def _generate_sbom_file(self, components: list) -> list: + def _generate_sbom_file(self, components: list) -> dict[str, list[dict[str, str]]]: """ Generate a list of PURLs for the SBOM file. :param components: List of undeclared components - :return: List of dictionaries containing PURLs + :return: SBOM Dictionary with components """ - sbom = {} + + unique_components = {} if components is None: self.print_stderr(f'WARNING: No components provided!') else: for component in components: - sbom[component['purl']] = { 'purl': component['purl'] } - return list(sbom.values()) + unique_components[component['purl']] = { 'purl': component['purl'] } + + sbom = { + 'components': list(unique_components.values()) + } + + return sbom def run(self): """ diff --git a/tests/policy-inspect-test.py b/tests/policy-inspect-test.py index 7497b941..3bfd897c 100644 --- a/tests/policy-inspect-test.py +++ b/tests/policy-inspect-test.py @@ -172,20 +172,22 @@ def test_undeclared_policy(self): expected_summary_output = """5 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json - [ - { - "purl": "pkg:github/scanoss/scanner.c" - }, - { - "purl": "pkg:github/scanoss/wfp" - }, - { - "purl": "pkg:npm/%40electron/rebuild" - }, - { - "purl": "pkg:npm/%40emotion/react" - } - ]``` + { + "components":[ + { + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + }, + { + "purl": "pkg:npm/%40electron/rebuild" + }, + { + "purl": "pkg:npm/%40emotion/react" + } + ] + }``` """ self.assertEqual(len(details['components']), 5) self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', @@ -215,21 +217,25 @@ def test_undeclared_policy_markdown(self): expected_summary_output = """5 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json - [ - { - "purl": "pkg:github/scanoss/scanner.c" - }, - { - "purl": "pkg:github/scanoss/wfp" - }, - { - "purl": "pkg:npm/%40electron/rebuild" - }, - { - "purl": "pkg:npm/%40emotion/react" - } - ]``` + { + "components":[ + { + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + }, + { + "purl": "pkg:npm/%40electron/rebuild" + }, + { + "purl": "pkg:npm/%40emotion/react" + } + ] + }``` """ + + print(summary) self.assertEqual(status, 0) self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output)) From 89b688dcca09b2546e75ae3872016e75b8a06f35 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:58:51 -0300 Subject: [PATCH 186/489] bug:SP-1777 Fixes return type in undeclared component's SBOM file method * bug:SP-1777 Fixes return type in undeclared component's SBOM file method --- CHANGELOG.md | 8 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 5 +- src/scanoss/inspection/copyleft.py | 2 +- src/scanoss/inspection/policy_check.py | 5 +- .../inspection/undeclared_component.py | 4 +- src/scanoss/inspection/utils/license_utils.py | 2 +- .../inspection/utils/markdown_utils.py | 23 ------ src/scanoss/inspection/utils/result_utils.py | 79 ------------------- tests/policy-inspect-test.py | 4 +- 10 files changed, 20 insertions(+), 114 deletions(-) delete mode 100644 src/scanoss/inspection/utils/markdown_utils.py delete mode 100644 src/scanoss/inspection/utils/result_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index efa79ec5..9932e5b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.17.3] - 2024-11-05 +### Fixed +- Fixed undeclared policy + + ## [1.17.2] - 2024-11-01 ### Fixed - Fixed parsing of dependencies in Policy Checks @@ -378,4 +383,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.16.0]: https://github.com/scanoss/scanoss.py/compare/v1.15.0...v1.16.0 [1.17.0]: https://github.com/scanoss/scanoss.py/compare/v1.16.0...v1.17.0 [1.17.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.0...v1.17.1 -[1.17.2]: https://github.com/scanoss/scanoss.py/compare/v1.17.1...v1.17.2 \ No newline at end of file +[1.17.2]: https://github.com/scanoss/scanoss.py/compare/v1.17.1...v1.17.2 +[1.17.3]: https://github.com/scanoss/scanoss.py/compare/v1.17.2...v1.17.3 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 6a2d87d3..bfe37a3a 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.17.2" +__version__ = "1.17.3" diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 16a3d8f0..9eadcaca 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -26,8 +26,9 @@ from pathlib import Path import sys import pypac -from scanoss.inspection.copyleft import Copyleft -from scanoss.inspection.undeclared_component import UndeclaredComponent + +from .inspection.copyleft import Copyleft +from .inspection.undeclared_component import UndeclaredComponent from .threadeddependencies import SCOPE from .scanoss_settings import ScanossSettings from .scancodedeps import ScancodeDeps diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index c99621d5..d7d6992f 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -23,7 +23,7 @@ """ import json from typing import Dict, Any -from scanoss.inspection.policy_check import PolicyCheck, PolicyStatus +from .policy_check import PolicyCheck, PolicyStatus class Copyleft(PolicyCheck): """ diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index fd9a369b..20b75f8a 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -26,8 +26,9 @@ from abc import abstractmethod from enum import Enum from typing import Callable, List, Dict, Any -from scanoss.inspection.utils.license_utils import LicenseUtil -from scanoss.scanossbase import ScanossBase +from .utils.license_utils import LicenseUtil +from ..scanossbase import ScanossBase + class PolicyStatus(Enum): """ diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index d111334f..f18ff148 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -23,7 +23,7 @@ """ import json from typing import Dict, Any -from scanoss.inspection.policy_check import PolicyCheck, PolicyStatus +from .policy_check import PolicyCheck, PolicyStatus class UndeclaredComponent(PolicyCheck): """ @@ -115,7 +115,7 @@ def _markdown(self, components: list) -> Dict[str,Any]: 'summary': self._get_summary(components), } - def _generate_sbom_file(self, components: list) -> dict[str, list[dict[str, str]]]: + def _generate_sbom_file(self, components: list) -> dict: """ Generate a list of PURLs for the SBOM file. diff --git a/src/scanoss/inspection/utils/license_utils.py b/src/scanoss/inspection/utils/license_utils.py index 9111d346..f97b2758 100644 --- a/src/scanoss/inspection/utils/license_utils.py +++ b/src/scanoss/inspection/utils/license_utils.py @@ -21,7 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from scanoss.scanossbase import ScanossBase +from ...scanossbase import ScanossBase DEFAULT_COPYLEFT_LICENSES = { 'agpl-3.0-only', 'artistic-1.0', 'artistic-2.0', 'cc-by-sa-4.0', 'cddl-1.0', 'cddl-1.1', 'cecill-2.1', diff --git a/src/scanoss/inspection/utils/markdown_utils.py b/src/scanoss/inspection/utils/markdown_utils.py deleted file mode 100644 index e4b9e902..00000000 --- a/src/scanoss/inspection/utils/markdown_utils.py +++ /dev/null @@ -1,23 +0,0 @@ -def generate_table(headers, rows, centered_columns=None): - """ - Generate Markdown table - :param headers: List of headers - :param rows: Rows - :param centered_columns: List with centered columns - """ - COL_SEP = ' | ' - centered_column_set = set(centered_columns or []) - def create_separator(header, index): - if centered_columns is None: - return '-' - return ':-:' if index in centered_column_set else '-' - - row_separator = COL_SEP + COL_SEP.join( - create_separator(header, index) for index, header in enumerate(headers) - ) + COL_SEP - - table_rows = [COL_SEP + COL_SEP.join(headers) + COL_SEP] - table_rows.append(row_separator) - table_rows.extend(COL_SEP + COL_SEP.join(row) + COL_SEP for row in rows) - - return '\n'.join(table_rows) \ No newline at end of file diff --git a/src/scanoss/inspection/utils/result_utils.py b/src/scanoss/inspection/utils/result_utils.py deleted file mode 100644 index 4d089924..00000000 --- a/src/scanoss/inspection/utils/result_utils.py +++ /dev/null @@ -1,79 +0,0 @@ -from enum import Enum -from typing import Dict, Any - -from scanoss.inspection.utils.license_utils import license_util - - -class ComponentID(Enum): - FILE = "file" - SNIPPET = "snippet" - DEPENDENCY = "dependency" - - -def _append_component(components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: - """ - Append a new component to the components dictionary. - - This function creates a new entry in the components dictionary for the given component, - or updates an existing entry if the component already exists. It also processes the - licenses associated with the component. - - :param components: The existing dictionary of components - :param new_component: The new component to be added or updated - :return: The updated components dictionary - """ - component_key = f"{new_component['purl'][0]}@{new_component['version']}" - components[component_key] = { - 'purl': new_component['purl'][0], - 'version': new_component['version'], - 'licenses': {}, - 'status': new_component['status'], - } - - # Process licenses for this component - for l in new_component['licenses']: - spdxid = l['name'] - components[component_key]['licenses'][spdxid] = { - 'spdxid': spdxid, - 'copyleft': license_util.is_copyleft(spdxid), - 'url': l.get('url') - } - - return components - - -def get_components(results: Dict[str, Any]) -> list: - """ - Process the results dictionary to extract and format component information. - - This function iterates through the results dictionary, identifying components from - different sources (files, snippets, and dependencies). It consolidates this information - into a list of unique components, each with its associated licenses and other details. - - :param results: A dictionary containing the raw results of a component scan - :return: A list of dictionaries, each representing a unique component with its details - """ - components = {} - for component in results.values(): - for c in component: - if c['id'] in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: - component_key = f"{c['purl'][0]}@{c['version']}" - - # Initialize or update the component entry - if component_key not in components: - components = _append_component(components, c) - - if c['id'] == ComponentID.DEPENDENCY.value: - for d in c['dependencies']: - component_key = f"{d['purl'][0]}@{d['version']}" - - if component_key not in components: - components = _append_component(components, d) - # End of for loop - # End if - # End if - results = list(components.values()) - for component in results: - component['licenses'] = list(component['licenses'].values()) - - return results diff --git a/tests/policy-inspect-test.py b/tests/policy-inspect-test.py index 3bfd897c..bd9e5cad 100644 --- a/tests/policy-inspect-test.py +++ b/tests/policy-inspect-test.py @@ -26,8 +26,8 @@ import re import unittest -from scanoss.inspection.copyleft import Copyleft -from scanoss.inspection.undeclared_component import UndeclaredComponent +from src.scanoss.inspection.copyleft import Copyleft +from src.scanoss.inspection.undeclared_component import UndeclaredComponent class MyTestCase(unittest.TestCase): From 3cade5d1817a9c1ead82793da2dc6b06b5fcb6b7 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 6 Nov 2024 16:20:55 +0100 Subject: [PATCH 187/489] Add devcontainer configuration for local development --- .devcontainer/Dockerfile.dev | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .devcontainer/Dockerfile.dev diff --git a/.devcontainer/Dockerfile.dev b/.devcontainer/Dockerfile.dev new file mode 100644 index 00000000..93518137 --- /dev/null +++ b/.devcontainer/Dockerfile.dev @@ -0,0 +1,26 @@ +FROM --platform=linux/arm64/v8 python:3.10-slim-buster + +RUN apt-get update && apt-get install -y \ + build-essential \ + gcc \ + && apt-get clean + +WORKDIR /workspace + +COPY requirements.txt requirements-dev.txt requirements-scancode.txt /tmp/ + +# Install Python dependencies +RUN pip3 install --no-cache-dir -r /tmp/requirements.txt && \ + pip3 install --no-cache-dir -r /tmp/requirements-dev.txt && \ + pip3 install --no-cache-dir scanoss_winnowing && \ + pip3 install --no-cache-dir scancode-toolkit-mini + +# Download compile and install typecode-libmagic from source (as there is not ARM wheel available) +ADD https://github.com/nexB/typecode_libmagic_from_sources/archive/refs/tags/v5.39.210212.tar.gz /install/ +RUN tar -xvzf /install/v5.39.210212.tar.gz -C /install \ + && cd /install/typecode_libmagic_from_sources* \ + && ./build.sh && python3 setup.py sdist bdist_wheel \ + && pip3 install --user `ls /install/typecode_libmagic_from_sources*/dist/*.whl` + +# Keep container running +CMD ["sleep", "infinity"] From 71e1a07a2b9ab8628e87118e4cac2f43c1f59cb2 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 6 Nov 2024 16:31:55 +0100 Subject: [PATCH 188/489] Add devcontainer.json, update readme --- .devcontainer/Dockerfile.dev | 2 -- .devcontainer/devcontainer.json | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile.dev b/.devcontainer/Dockerfile.dev index 93518137..b8d6cdf7 100644 --- a/.devcontainer/Dockerfile.dev +++ b/.devcontainer/Dockerfile.dev @@ -9,7 +9,6 @@ WORKDIR /workspace COPY requirements.txt requirements-dev.txt requirements-scancode.txt /tmp/ -# Install Python dependencies RUN pip3 install --no-cache-dir -r /tmp/requirements.txt && \ pip3 install --no-cache-dir -r /tmp/requirements-dev.txt && \ pip3 install --no-cache-dir scanoss_winnowing && \ @@ -22,5 +21,4 @@ RUN tar -xvzf /install/v5.39.210212.tar.gz -C /install \ && ./build.sh && python3 setup.py sdist bdist_wheel \ && pip3 install --user `ls /install/typecode_libmagic_from_sources*/dist/*.whl` -# Keep container running CMD ["sleep", "infinity"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..e99f7857 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,41 @@ +{ + "name": "SCANOSS Dev Container", + "build": { + "dockerfile": "Dockerfile.dev", + "context": "..", + "args": { + "BUILDPLATFORM": "linux/arm64" + } + }, + "runArgs": ["--platform=linux/arm64"], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.black-formatter", + "ms-python.mypy-type-checker", + "github.copilot" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.formatting.provider": "black", + "python.linting.enabled": true, + "python.linting.mypyEnabled": true, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "always" + } + } + } + }, + "remoteUser": "root", + "features": { + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + "mounts": [ + "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached" + ], + "postCreateCommand": "pip install -e ." +} From 27c42afaa8803a453fd0829bd9b0b8884a6b718b Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 8 Nov 2024 06:50:40 +0100 Subject: [PATCH 189/489] Update dockerfile.dev --- .devcontainer/Dockerfile.dev | 2 +- .devcontainer/devcontainer.json | 2 +- README.md | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile.dev b/.devcontainer/Dockerfile.dev index b8d6cdf7..f188e6f4 100644 --- a/.devcontainer/Dockerfile.dev +++ b/.devcontainer/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM --platform=linux/arm64/v8 python:3.10-slim-buster +FROM python:3.10-slim-buster RUN apt-get update && apt-get install -y \ build-essential \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e99f7857..70698ab7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -37,5 +37,5 @@ "mounts": [ "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached" ], - "postCreateCommand": "pip install -e ." + "postCreateCommand": "make dev_setup" } diff --git a/README.md b/README.md index 4d164312..7a395e26 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,17 @@ To enable dependency scanning, an extra tool is required: scancode-toolkit pip3 install -r requirements-scancode.txt ``` +### Devcontainer Setup +To simplify the development environment setup, a devcontainer configuration is provided. This allows you to develop inside a containerized environment with all necessary dependencies pre-installed. + +To use the devcontainer setup: +1. Install [Visual Studio Code](https://code.visualstudio.com/). +2. Install the [Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension. +3. Open the project in Visual Studio Code. +4. When prompted, reopen the project in the container. + +This will build the container defined in the `.devcontainer` folder and open a new Visual Studio Code window connected to the container. + ### Package Development More details on Python packaging/distribution can be found [here](https://packaging.python.org/overview/), [here](https://packaging.python.org/guides/distributing-packages-using-setuptools/), and [here](https://packaging.python.org/guides/using-testpypi/#using-test-pypi). From f42bf174a0d489d2d9a831496c8d4ef7039377f6 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 8 Nov 2024 10:23:17 +0100 Subject: [PATCH 190/489] Handle parsing of scan/dependency responses by merging them into a unified dictionary --- src/scanoss/scanner.py | 106 +++++++++++++++++------------- "src/shaders/compileshaders\".py" | 90 +++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 47 deletions(-) create mode 100644 "src/shaders/compileshaders\".py" diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index f052ee07..c63e1fc2 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -25,6 +25,7 @@ import os import sys import datetime +from typing import Any, Dict, List, Optional import importlib_resources from progress.bar import Bar @@ -490,66 +491,41 @@ def __run_scan_threaded(self, scan_started: bool, file_count: int) -> bool: success = False return success - def __finish_scan_threaded(self, file_map: dict = None) -> bool: - """ - Wait for the threaded scans to complete - :param file_map: mapping of obfuscated files back into originals - :return: True if successful, False otherwise + def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> bool: + """Wait for the threaded scan to complete and process the results + + Args: + file_map: Mapping of obfuscated files back to originals + + Returns: + bool: True if successful, False otherwise + + Raises: + ValueError: If output format is invalid """ - success = True - responses = None + success: bool = True + scan_responses = None dep_responses = None if self.is_file_or_snippet_scan(): if not self.threaded_scan.complete(): # Wait for the scans to complete self.print_stderr(f'Warning: Scanning analysis ran into some trouble.') success = False self.threaded_scan.complete_bar() - responses = self.threaded_scan.responses + scan_responses = self.threaded_scan.responses if self.is_dependency_scan(): self.print_msg('Retrieving dependency data...') if not self.threaded_deps.complete(): - self.print_stderr(f'Warning: Dependency analysis ran into some trouble.') + self.print_stderr( + f'Warning: Dependency analysis ran into some trouble.' + ) success = False dep_responses = self.threaded_deps.responses - # self.print_stderr(f'Dep Data: {dep_responses}') - # TODO change to dictionary - raw_output = "{\n" - # TODO look into merging the two dictionaries. See https://favtutor.com/blogs/merge-dictionaries-python - if responses or dep_responses: - first = True - if responses: - for scan_resp in responses: - if scan_resp is not None: - for key, value in scan_resp.items(): - if file_map: # We have a map for obfuscated files. Check if we can revert it - fm = file_map.get(key) - if fm: - key = fm # Replace the obfuscated filename - if first: - raw_output += " \"%s\":%s" % (key, json.dumps(value, indent=2)) - first = False - else: - raw_output += ",\n \"%s\":%s" % (key, json.dumps(value, indent=2)) - # End for loop - if dep_responses: - dep_files = dep_responses.get("files") - if dep_files and len(dep_files) > 0: - for dep_file in dep_files: - file = dep_file.pop("file", None) - if file is not None: - if first: - raw_output += " \"%s\":[%s]" % (file, json.dumps(dep_file, indent=2)) - first = False - else: - raw_output += ",\n \"%s\":[%s]" % (file, json.dumps(dep_file, indent=2)) - # End for loop - raw_output += "\n}" - try: - raw_results = json.loads(raw_output) - except Exception as e: - raise Exception(f'ERROR: Problem decoding parsed json: {e}') - results = self.post_processor.load_results(raw_results).post_process() + raw_scan_results = self._merge_scan_results( + scan_responses, dep_responses, file_map + ) + + results = self.post_processor.load_results(raw_scan_results).post_process() if self.output_format == 'plain': self.__log_result(json.dumps(results, indent=2, sort_keys=True)) @@ -567,6 +543,42 @@ def __finish_scan_threaded(self, file_map: dict = None) -> bool: success = False return success + def _merge_scan_results( + self, + scan_responses: list | None, + dep_responses: dict | None, + file_map: dict | None, + ) -> Dict[str, Any]: + """Merge scan and dependency responses into a single dictionary""" + results: Dict[str, Any] = {} + + if scan_responses: + for response in scan_responses: + if response is not None: + if file_map: + response = self._deobfuscate_filenames(response, file_map) + results.update(response) + + dep_files = dep_responses.get("files", None) if dep_responses else None + if dep_files: + for dep_file in dep_files: + file = dep_file.pop("file", None) + if file: + results[file] = dep_file + + return results + + def _deobfuscate_filenames(self, response: dict, file_map: dict) -> dict: + """Convert obfuscated filenames back to original names""" + deobfuscated = {} + for key, value in response.items(): + deobfuscated_name = file_map.get(key, None) + if deobfuscated_name: + deobfuscated[deobfuscated_name] = value + else: + deobfuscated[key] = value + return deobfuscated + def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dict = None, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None) -> bool: """ diff --git "a/src/shaders/compileshaders\".py" "b/src/shaders/compileshaders\".py" new file mode 100644 index 00000000..0e093f26 --- /dev/null +++ "b/src/shaders/compileshaders\".py" @@ -0,0 +1,90 @@ +# Copyright 2020 Google LLC +# Copyright 2023-2024 Sascha Willems + +import argparse +import fileinput +import os +import subprocess +import sys + +parser = argparse.ArgumentParser(description='Compile all .hlsl shaders') +parser.add_argument('--dxc', type=str, help='path to DXC executable') +args = parser.parse_args() + +def findDXC(): + def isExe(path): + return os.path.isfile(path) and os.access(path, os.X_OK) + + if args.dxc != None and isExe(args.dxc): + return args.dxc + + exe_name = "dxc" + if os.name == "nt": + exe_name += ".exe" + + for exe_dir in os.environ["PATH"].split(os.pathsep): + full_path = os.path.join(exe_dir, exe_name) + if isExe(full_path): + return full_path + + sys.exit("Could not find DXC executable on PATH, and was not specified with --dxc") + +file_extensions = tuple([".vert", ".frag", ".comp", ".geom", ".tesc", ".tese", ".rgen", ".rchit", ".rmiss", ".mesh", ".task"]) + +dxc_path = findDXC() +dir_path = os.path.dirname(os.path.realpath(__file__)) +dir_path = dir_path.replace('\\', '/') +for root, dirs, files in os.walk(dir_path): + for file in files: + if file.endswith(file_extensions): + hlsl_file = os.path.join(root, file) + spv_out = hlsl_file + ".spv" + + target = '' + profile = '' + additional_exts = '' + if(hlsl_file.find('.vert') != -1): + profile = 'vs_6_1' + elif(hlsl_file.find('.frag') != -1): + profile = 'ps_6_4' + elif(hlsl_file.find('.comp') != -1): + profile = 'cs_6_1' + elif(hlsl_file.find('.geom') != -1): + profile = 'gs_6_1' + elif(hlsl_file.find('.tesc') != -1): + profile = 'hs_6_1' + elif(hlsl_file.find('.tese') != -1): + profile = 'ds_6_1' + elif(hlsl_file.find('.rgen') != -1 or + hlsl_file.find('.rchit') != -1 or + hlsl_file.find('.rmiss') != -1): + target='-fspv-target-env=vulkan1.2' + profile = 'lib_6_3' + elif(hlsl_file.find('.mesh') != -1): + target='-fspv-target-env=vulkan1.2' + additional_exts = '-fspv-extension=SPV_EXT_mesh_shader' + profile = 'ms_6_6' + elif(hlsl_file.find('.task') != -1): + target='-fspv-target-env=vulkan1.2' + additional_exts = '-fspv-extension=SPV_EXT_mesh_shader' + profile = 'as_6_6' + + if root.endswith("debugprintf"): + additional_exts = '-fspv-extension=SPV_KHR_non_semantic_info' + + print('Compiling %s' % (hlsl_file)) + subprocess.check_output([ + dxc_path, + '-spirv', + '-T', profile, + '-E', 'main', + '-fspv-extension=SPV_KHR_ray_tracing', + '-fspv-extension=SPV_KHR_multiview', + '-fspv-extension=SPV_KHR_shader_draw_parameters', + '-fspv-extension=SPV_EXT_descriptor_indexing', + '-fspv-extension=SPV_KHR_ray_query', + '-fspv-extension=SPV_KHR_fragment_shading_rate', + additional_exts, + target, + hlsl_file, + '-Fo', spv_out]) From 25208d7306b84e7277f73a14c6038b95d4cf32f1 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 8 Nov 2024 09:24:58 +0000 Subject: [PATCH 191/489] Remove test file --- "src/shaders/compileshaders\".py" | 90 ------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 "src/shaders/compileshaders\".py" diff --git "a/src/shaders/compileshaders\".py" "b/src/shaders/compileshaders\".py" deleted file mode 100644 index 0e093f26..00000000 --- "a/src/shaders/compileshaders\".py" +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2020 Google LLC -# Copyright 2023-2024 Sascha Willems - -import argparse -import fileinput -import os -import subprocess -import sys - -parser = argparse.ArgumentParser(description='Compile all .hlsl shaders') -parser.add_argument('--dxc', type=str, help='path to DXC executable') -args = parser.parse_args() - -def findDXC(): - def isExe(path): - return os.path.isfile(path) and os.access(path, os.X_OK) - - if args.dxc != None and isExe(args.dxc): - return args.dxc - - exe_name = "dxc" - if os.name == "nt": - exe_name += ".exe" - - for exe_dir in os.environ["PATH"].split(os.pathsep): - full_path = os.path.join(exe_dir, exe_name) - if isExe(full_path): - return full_path - - sys.exit("Could not find DXC executable on PATH, and was not specified with --dxc") - -file_extensions = tuple([".vert", ".frag", ".comp", ".geom", ".tesc", ".tese", ".rgen", ".rchit", ".rmiss", ".mesh", ".task"]) - -dxc_path = findDXC() -dir_path = os.path.dirname(os.path.realpath(__file__)) -dir_path = dir_path.replace('\\', '/') -for root, dirs, files in os.walk(dir_path): - for file in files: - if file.endswith(file_extensions): - hlsl_file = os.path.join(root, file) - spv_out = hlsl_file + ".spv" - - target = '' - profile = '' - additional_exts = '' - if(hlsl_file.find('.vert') != -1): - profile = 'vs_6_1' - elif(hlsl_file.find('.frag') != -1): - profile = 'ps_6_4' - elif(hlsl_file.find('.comp') != -1): - profile = 'cs_6_1' - elif(hlsl_file.find('.geom') != -1): - profile = 'gs_6_1' - elif(hlsl_file.find('.tesc') != -1): - profile = 'hs_6_1' - elif(hlsl_file.find('.tese') != -1): - profile = 'ds_6_1' - elif(hlsl_file.find('.rgen') != -1 or - hlsl_file.find('.rchit') != -1 or - hlsl_file.find('.rmiss') != -1): - target='-fspv-target-env=vulkan1.2' - profile = 'lib_6_3' - elif(hlsl_file.find('.mesh') != -1): - target='-fspv-target-env=vulkan1.2' - additional_exts = '-fspv-extension=SPV_EXT_mesh_shader' - profile = 'ms_6_6' - elif(hlsl_file.find('.task') != -1): - target='-fspv-target-env=vulkan1.2' - additional_exts = '-fspv-extension=SPV_EXT_mesh_shader' - profile = 'as_6_6' - - if root.endswith("debugprintf"): - additional_exts = '-fspv-extension=SPV_KHR_non_semantic_info' - - print('Compiling %s' % (hlsl_file)) - subprocess.check_output([ - dxc_path, - '-spirv', - '-T', profile, - '-E', 'main', - '-fspv-extension=SPV_KHR_ray_tracing', - '-fspv-extension=SPV_KHR_multiview', - '-fspv-extension=SPV_KHR_shader_draw_parameters', - '-fspv-extension=SPV_EXT_descriptor_indexing', - '-fspv-extension=SPV_KHR_ray_query', - '-fspv-extension=SPV_KHR_fragment_shading_rate', - additional_exts, - target, - hlsl_file, - '-Fo', spv_out]) From a64ffa0ed1f594c3373f10a64a28230e72f65791 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 8 Nov 2024 13:14:07 +0100 Subject: [PATCH 192/489] Replace backslashes on windows for obfuscated files --- src/scanoss/winnowing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 9c21930d..1afc6a33 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -29,6 +29,7 @@ """ import hashlib import pathlib +import platform import re from crc32c import crc32c @@ -310,6 +311,8 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: wfp_filename = repr(file).strip("'") # return a utf-8 compatible version of the filename if self.obfuscate: # hide the real size of the file and its name, but keep the suffix wfp_filename = f'{self.ob_count}{pathlib.Path(file).suffix}' + if platform.system() == 'Windows': + wfp_filename = wfp_filename.replace('\\', '/') self.ob_count = self.ob_count + 1 self.file_map[wfp_filename] = file # Save the file name map for later (reverse lookup) From 6ca0d64cba4120b49ab46b7f16a2765249ee58c2 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 8 Nov 2024 13:48:20 +0100 Subject: [PATCH 193/489] Replace also file name when obfuscate --- src/scanoss/winnowing.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 1afc6a33..b669ddf1 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -309,12 +309,17 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # Print file line content_length = len(contents) wfp_filename = repr(file).strip("'") # return a utf-8 compatible version of the filename - if self.obfuscate: # hide the real size of the file and its name, but keep the suffix + if ( + self.obfuscate + ): # hide the real size of the file and its name, but keep the suffix wfp_filename = f'{self.ob_count}{pathlib.Path(file).suffix}' - if platform.system() == 'Windows': - wfp_filename = wfp_filename.replace('\\', '/') + if platform.system() == "Windows": + wfp_filename = wfp_filename.replace("\\", "/") + file = file.replace("\\", "/") self.ob_count = self.ob_count + 1 - self.file_map[wfp_filename] = file # Save the file name map for later (reverse lookup) + self.file_map[wfp_filename] = ( + file # Save the file name map for later (reverse lookup) + ) wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename) # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip @@ -467,7 +472,7 @@ def crc8_buffer(self, buffer): crc = self.crc8_byte(crc, buffer[index]) crc ^= CRC8_MAXIM_DOW_FINAL # Bitwise OR (XOR) of crc in Maxim Dow Final return crc - + # # End of Winnowing Class # From 0d0709b413a2b01918c40497f912edf24711d3f6 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 8 Nov 2024 13:51:49 +0100 Subject: [PATCH 194/489] Update version and changelog --- CHANGELOG.md | 4 ++++ src/scanoss/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9932e5b6..a4047228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Upcoming changes... + +## [1.17.4] - 2024-11-08 +### Fixed +- Fix backslashes in file paths on Windows ## [1.17.3] - 2024-11-05 ### Fixed diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index bfe37a3a..88a5b159 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.17.3" +__version__ = "1.17.4" From 0acfc8163a6270e01280a3c252812ecacd4d55f0 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 8 Nov 2024 18:17:58 +0100 Subject: [PATCH 195/489] Update windows filename --- src/scanoss/scanner.py | 6 +++--- src/scanoss/winnowing.py | 19 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index c63e1fc2..ea9b08d1 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -545,9 +545,9 @@ def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> b def _merge_scan_results( self, - scan_responses: list | None, - dep_responses: dict | None, - file_map: dict | None, + scan_responses: Optional[List], + dep_responses: Optional[Dict[str,Any]], + file_map: Optional[Dict[str, Any]], ) -> Dict[str, Any]: """Merge scan and dependency responses into a single dictionary""" results: Dict[str, Any] = {} diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index b669ddf1..4026ada6 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -308,18 +308,15 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: return '' # Print file line content_length = len(contents) - wfp_filename = repr(file).strip("'") # return a utf-8 compatible version of the filename - if ( - self.obfuscate - ): # hide the real size of the file and its name, but keep the suffix - wfp_filename = f'{self.ob_count}{pathlib.Path(file).suffix}' - if platform.system() == "Windows": - wfp_filename = wfp_filename.replace("\\", "/") - file = file.replace("\\", "/") + original_filename = file + + if platform.system() == 'Windows': + original_filename = file.replace('\\', '/') + wfp_filename = repr(original_filename).strip("'") # return a utf-8 compatible version of the filename + if self.obfuscate: # hide the real size of the file and its name, but keep the suffix + wfp_filename = f'{self.ob_count}{pathlib.Path(original_filename).suffix}' self.ob_count = self.ob_count + 1 - self.file_map[wfp_filename] = ( - file # Save the file name map for later (reverse lookup) - ) + self.file_map[wfp_filename] = original_filename # Save the file name map for later (reverse lookup) wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename) # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip From 0fb7eca1759642527c6d0862d34c4127c1f03fee Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 11 Nov 2024 15:05:15 +0100 Subject: [PATCH 196/489] Add devcontainer.json to gitignore, add devcontainer config example, update readme --- .../{Dockerfile.dev => Containerfile.dev} | 0 .devcontainer/devcontainer.json | 41 ------------------- .gitignore | 3 +- README.md | 7 +++- 4 files changed, 8 insertions(+), 43 deletions(-) rename .devcontainer/{Dockerfile.dev => Containerfile.dev} (100%) delete mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile.dev b/.devcontainer/Containerfile.dev similarity index 100% rename from .devcontainer/Dockerfile.dev rename to .devcontainer/Containerfile.dev diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 70698ab7..00000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "SCANOSS Dev Container", - "build": { - "dockerfile": "Dockerfile.dev", - "context": "..", - "args": { - "BUILDPLATFORM": "linux/arm64" - } - }, - "runArgs": ["--platform=linux/arm64"], - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance", - "ms-python.black-formatter", - "ms-python.mypy-type-checker", - "github.copilot" - ], - "settings": { - "python.defaultInterpreterPath": "/usr/local/bin/python", - "python.formatting.provider": "black", - "python.linting.enabled": true, - "python.linting.mypyEnabled": true, - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": "always" - } - } - } - }, - "remoteUser": "root", - "features": { - "ghcr.io/devcontainers/features/git:1": {}, - "ghcr.io/devcontainers/features/github-cli:1": {} - }, - "mounts": [ - "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached" - ], - "postCreateCommand": "make dev_setup" -} diff --git a/.gitignore b/.gitignore index f6ba5eef..778851f0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ bad*.txt *.gz *.zip local-*.txt -docs/build \ No newline at end of file +docs/build +.devcontainer/devcontainer.json \ No newline at end of file diff --git a/README.md b/README.md index 7a395e26..a95f6277 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,12 @@ To use the devcontainer setup: 1. Install [Visual Studio Code](https://code.visualstudio.com/). 2. Install the [Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension. 3. Open the project in Visual Studio Code. -4. When prompted, reopen the project in the container. +4. Run +```bash +cp .devcontainer/devcontainer.example.json .devcontainer/devcontainer.json +``` +5. Update the `devcontainer.json` file with the desired settings. +6. When prompted, reopen the project in the container. This will build the container defined in the `.devcontainer` folder and open a new Visual Studio Code window connected to the container. From bb8dac1a9bd65a33461a46ff45cbbb16a2b9c900 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 12 Nov 2024 10:22:18 +0100 Subject: [PATCH 197/489] bug: Fix dependencies scan result structure --- CHANGELOG.md | 8 +++++++- src/scanoss/__init__.py | 2 +- src/scanoss/scanner.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4047228..1375f5ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Upcoming changes... + +## [1.17.5] - 2024-11-12 +### Fixed +- Fix dependencies scan result structure ## [1.17.4] - 2024-11-08 ### Fixed @@ -388,4 +392,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.17.0]: https://github.com/scanoss/scanoss.py/compare/v1.16.0...v1.17.0 [1.17.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.0...v1.17.1 [1.17.2]: https://github.com/scanoss/scanoss.py/compare/v1.17.1...v1.17.2 -[1.17.3]: https://github.com/scanoss/scanoss.py/compare/v1.17.2...v1.17.3 \ No newline at end of file +[1.17.3]: https://github.com/scanoss/scanoss.py/compare/v1.17.2...v1.17.3 +[1.17.4]: https://github.com/scanoss/scanoss.py/compare/v1.17.3...v1.17.4 +[1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 88a5b159..4c385f65 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.17.4" +__version__ = '1.17.5' diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index ea9b08d1..7ad5932d 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -564,7 +564,7 @@ def _merge_scan_results( for dep_file in dep_files: file = dep_file.pop("file", None) if file: - results[file] = dep_file + results[file] = [dep_file] return results From 354a0555b7bee51f05798dd2547988ac5bcfaab7 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 12 Nov 2024 11:35:35 +0100 Subject: [PATCH 198/489] Add devcontainer.example.json --- .devcontainer/devcontainer.example.json | 42 +++++++++++++++++++++++++ .gitignore | 3 +- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/devcontainer.example.json diff --git a/.devcontainer/devcontainer.example.json b/.devcontainer/devcontainer.example.json new file mode 100644 index 00000000..496f43d6 --- /dev/null +++ b/.devcontainer/devcontainer.example.json @@ -0,0 +1,42 @@ +{ + "name": "SCANOSS Dev Container", + "build": { + "dockerfile": "Containerfile.dev", + "context": ".." + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.black-formatter", + "ms-python.isort", + "ms-python.flake8" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.formatting.provider": "black", + "python.formatting.blackPath": "/usr/local/bin/black", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.pylintEnabled": false, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + }, + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false + } + } + }, + "mounts": [ + "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached" + ], + "postCreateCommand": "make dev_setup", + "features": { + "ghcr.io/devcontainers/features/python:1": { + "version": "3.11" + }, + "ghcr.io/devcontainers/features/git:1": {} + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 778851f0..361ada38 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ bad*.txt *.zip local-*.txt docs/build -.devcontainer/devcontainer.json \ No newline at end of file +.devcontainer/devcontainer.json +!.devcontainer/*.example.json From 0ae04ce266002a531f78c4cf4c26fb462e20f924 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 11 Nov 2024 12:19:21 +0100 Subject: [PATCH 199/489] feat: SP-1801 Add replace action to scan post processor --- src/scanoss/scanoss_settings.py | 10 +++ src/scanoss/scanpostprocessor.py | 143 ++++++++++++++++++++++++------- 2 files changed, 121 insertions(+), 32 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 21230295..a6a9bfff 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -151,6 +151,16 @@ def get_bom_remove(self) -> List[BomEntry]: return self._get_bom() return self._get_bom().get("remove", []) + def get_bom_replace(self) -> List[BomEntry]: + """Get the list of components to replace in the scan + + Returns: + list: List of components to replace in the scan + """ + if self.settings_file_type == "legacy": + return [] + return self._get_bom().get("replace", []) + def get_sbom(self): """Get the SBOM to be sent to the SCANOSS API diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 6f1e9b0e..f58c4db3 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -1,28 +1,28 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ -from typing import List +from typing import List, Tuple from .scanoss_settings import BomEntry, ScanossSettings from .scanossbase import ScanossBase @@ -67,6 +67,7 @@ def post_process(self): dict: Processed results """ self.remove_dismissed_files() + self.replace_purls() return self.results def remove_dismissed_files(self): @@ -75,7 +76,7 @@ def remove_dismissed_files(self): to_remove_entries = self.scan_settings.get_bom_remove() if not to_remove_entries: - return + self.results = self.results self.results = { result_path: result @@ -83,16 +84,94 @@ def remove_dismissed_files(self): if not self._should_remove_result(result_path, result, to_remove_entries) } + def replace_purls(self): + """Replace purls in the results based on the SCANOSS settings file""" + + to_replace_entries = self.scan_settings.get_bom_replace() + + if not to_replace_entries: + self.results = self.results + + for result_path, result in self.results.items(): + result = result[0] if isinstance(result, list) else result + should_replace, to_replace_with = self._should_replace_result( + result_path, result, to_replace_entries + ) + if should_replace: + result["purl"] = [to_replace_with] + + def _should_replace_result(self, result_path: str, result: dict, to_replace_entries: List[BomEntry]) -> Tuple[bool, str]: + """Check if a result should be replaced based on the SCANOSS settings + + Args: + result_path (str): Path of the result + result (dict): Result to check + to_replace_entries (List[BomEntry]): BOM entries to replace from the settings file + + Returns: + bool: True if the result should be replaced, False otherwise + str: The purl to replace with + """ + result_purls = result.get('purl', []) + + for to_replace_entry in to_replace_entries: + to_replace_path = to_replace_entry.get('path') + to_replace_purl = to_replace_entry.get('purl') + to_replace_with = to_replace_entry.get('replace_with') + + if not to_replace_path and not to_replace_purl or not to_replace_with: + continue + + # If it's the same purl as the one in the result, skip fast + if to_replace_with in result_purls: + continue + + # Bom entry has both path and purl + if self._is_full_match(result_path, result_purls, to_replace_entry): + self._print_replacement_message(result_path, result_purls, to_replace_entry) + return True, to_replace_with + + # Bom entry has only purl + if not to_replace_path and to_replace_purl in result_purls: + self._print_replacement_message(result_path, result_purls, to_replace_entry) + return True, to_replace_with + + # Bom entry has only path + if not to_replace_purl and to_replace_path == result_path: + self._print_removal_message(result_path, result_purls, to_replace_entry) + return True, to_replace_with + + return False, None + + def _print_replacement_message( + self, result_path: str, result_purls: List[str], to_replace_entry: BomEntry + ) -> None: + """Print a message about replacing a result""" + if to_replace_entry.get("path") and to_replace_entry.get('purl'): + message = f"Replacing purl for '{result_path}'. Full match found." + elif to_replace_entry.get('purl'): + message = f"Replacing purl for '{result_path}'. Found PURL match." + else: + message = f"Replacing purl for '{result_path}'. Found path match." + + self.print_debug( + f"{message}\n" + f"Details:\n" + f" - PURLs: {', '.join(result_purls)}\n" + f" - Path: '{result_path}'\n" + f" - Replacing with: '{to_replace_entry.get('replace_with')}'\n" + ) + def _should_remove_result( self, result_path: str, result: dict, to_remove_entries: List[BomEntry] ) -> bool: """Check if a result should be removed based on the SCANOSS settings""" result = result[0] if isinstance(result, list) else result - result_purls = result.get("purl", []) + result_purls = result.get('purl', []) for to_remove_entry in to_remove_entries: - to_remove_path = to_remove_entry.get("path") - to_remove_purl = to_remove_entry.get("purl") + to_remove_path = to_remove_entry.get('path') + to_remove_purl = to_remove_entry.get('purl') if not to_remove_path and not to_remove_purl: continue @@ -118,14 +197,14 @@ def _print_removal_message( self, result_path: str, result_purls: List[str], to_remove_entry: BomEntry ) -> None: """Print a message about removing a result""" - if to_remove_entry.get("path") and to_remove_entry.get("purl"): + if to_remove_entry.get("path") and to_remove_entry.get('purl'): message = f"Removing '{result_path}' from the results. Full match found." - elif to_remove_entry.get("purl"): + elif to_remove_entry.get('purl'): message = f"Removing '{result_path}' from the results. Found PURL match." else: message = f"Removing '{result_path}' from the results. Found path match." - self.print_msg( + self.print_debug( f"{message}\n" f"Details:\n" f" - PURLs: {', '.join(result_purls)}\n" @@ -153,7 +232,7 @@ def _is_full_match( return False return bool( - (bom_entry.get("purl") and bom_entry.get("path")) - and (bom_entry.get("path") == result_path) - and (bom_entry.get("purl") in result_purls) + (bom_entry.get('purl') and bom_entry.get('path')) + and (bom_entry.get('path') == result_path) + and (bom_entry.get('purl') in result_purls) ) From ecc38017e90037883130d68c86b9a8a1e0a5344d Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 11 Nov 2024 12:38:27 +0100 Subject: [PATCH 200/489] feat: SP-1801 Fixed linting, refactor duplicated code --- src/scanoss/scanpostprocessor.py | 106 ++++++++++--------------------- 1 file changed, 35 insertions(+), 71 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index f58c4db3..208d0ce3 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -49,7 +49,7 @@ def __init__( """ super().__init__(debug, trace, quiet) self.scan_settings = scan_settings - self.results = results + self.results: dict = results def load_results(self, raw_results: dict): """Load the raw results @@ -72,11 +72,9 @@ def post_process(self): def remove_dismissed_files(self): """Remove entries from the results based on files and/or purls specified in the SCANOSS settings file""" - to_remove_entries = self.scan_settings.get_bom_remove() - if not to_remove_entries: - self.results = self.results + return self.results = { result_path: result @@ -86,21 +84,19 @@ def remove_dismissed_files(self): def replace_purls(self): """Replace purls in the results based on the SCANOSS settings file""" - to_replace_entries = self.scan_settings.get_bom_replace() - if not to_replace_entries: - self.results = self.results + return for result_path, result in self.results.items(): result = result[0] if isinstance(result, list) else result - should_replace, to_replace_with = self._should_replace_result( - result_path, result, to_replace_entries - ) + should_replace, to_replace_with = self._should_replace_result(result_path, result, to_replace_entries) if should_replace: - result["purl"] = [to_replace_with] + result['purl'] = [to_replace_with] - def _should_replace_result(self, result_path: str, result: dict, to_replace_entries: List[BomEntry]) -> Tuple[bool, str]: + def _should_replace_result( + self, result_path: str, result: dict, to_replace_entries: List[BomEntry] + ) -> Tuple[bool, str]: """Check if a result should be replaced based on the SCANOSS settings Args: @@ -113,7 +109,6 @@ def _should_replace_result(self, result_path: str, result: dict, to_replace_entr str: The purl to replace with """ result_purls = result.get('purl', []) - for to_replace_entry in to_replace_entries: to_replace_path = to_replace_entry.get('path') to_replace_purl = to_replace_entry.get('purl') @@ -121,50 +116,21 @@ def _should_replace_result(self, result_path: str, result: dict, to_replace_entr if not to_replace_path and not to_replace_purl or not to_replace_with: continue - - # If it's the same purl as the one in the result, skip fast + if to_replace_with in result_purls: continue - # Bom entry has both path and purl - if self._is_full_match(result_path, result_purls, to_replace_entry): - self._print_replacement_message(result_path, result_purls, to_replace_entry) - return True, to_replace_with - - # Bom entry has only purl - if not to_replace_path and to_replace_purl in result_purls: - self._print_replacement_message(result_path, result_purls, to_replace_entry) - return True, to_replace_with - - # Bom entry has only path - if not to_replace_purl and to_replace_path == result_path: - self._print_removal_message(result_path, result_purls, to_replace_entry) + if ( + self._is_full_match(result_path, result_purls, to_replace_entry) + or (not to_replace_path and to_replace_purl in result_purls) + or (not to_replace_purl and to_replace_path == result_path) + ): + self._print_message(result_path, result_purls, to_replace_entry, 'Replacing') return True, to_replace_with return False, None - - def _print_replacement_message( - self, result_path: str, result_purls: List[str], to_replace_entry: BomEntry - ) -> None: - """Print a message about replacing a result""" - if to_replace_entry.get("path") and to_replace_entry.get('purl'): - message = f"Replacing purl for '{result_path}'. Full match found." - elif to_replace_entry.get('purl'): - message = f"Replacing purl for '{result_path}'. Found PURL match." - else: - message = f"Replacing purl for '{result_path}'. Found path match." - - self.print_debug( - f"{message}\n" - f"Details:\n" - f" - PURLs: {', '.join(result_purls)}\n" - f" - Path: '{result_path}'\n" - f" - Replacing with: '{to_replace_entry.get('replace_with')}'\n" - ) - def _should_remove_result( - self, result_path: str, result: dict, to_remove_entries: List[BomEntry] - ) -> bool: + def _should_remove_result(self, result_path: str, result: dict, to_remove_entries: List[BomEntry]) -> bool: """Check if a result should be removed based on the SCANOSS settings""" result = result[0] if isinstance(result, list) else result result_purls = result.get('purl', []) @@ -176,39 +142,37 @@ def _should_remove_result( if not to_remove_path and not to_remove_purl: continue - # Bom entry has both path and purl - if self._is_full_match(result_path, result_purls, to_remove_entry): - self._print_removal_message(result_path, result_purls, to_remove_entry) - return True - - # Bom entry has only purl - if not to_remove_path and to_remove_purl in result_purls: - self._print_removal_message(result_path, result_purls, to_remove_entry) - return True - - # Bom entry has only path - if not to_remove_purl and to_remove_path == result_path: - self._print_removal_message(result_path, result_purls, to_remove_entry) + if ( + self._is_full_match(result_path, result_purls, to_remove_entry) + or (not to_remove_path and to_remove_purl in result_purls) + or (not to_remove_purl and to_remove_path == result_path) + ): + self._print_message(result_path, result_purls, to_remove_entry, 'Removing') return True return False - def _print_removal_message( - self, result_path: str, result_purls: List[str], to_remove_entry: BomEntry + def _print_message( + self, + result_path: str, + result_purls: List[str], + bom_entry: BomEntry, + action: str, ) -> None: - """Print a message about removing a result""" - if to_remove_entry.get("path") and to_remove_entry.get('purl'): - message = f"Removing '{result_path}' from the results. Full match found." - elif to_remove_entry.get('purl'): - message = f"Removing '{result_path}' from the results. Found PURL match." + """Print a message about replacing or removing a result""" + if bom_entry.get('path') and bom_entry.get('purl'): + message = f"{action} '{result_path}'. Full match found." + elif bom_entry.get('purl'): + message = f"{action} '{result_path}'. Found PURL match." else: - message = f"Removing '{result_path}' from the results. Found path match." + message = f"{action} '{result_path}'. Found path match." self.print_debug( f"{message}\n" f"Details:\n" f" - PURLs: {', '.join(result_purls)}\n" f" - Path: '{result_path}'\n" + f" - {action} with: '{bom_entry.get('replace_with')}'\n" if action == 'Replacing' else '' ) def _is_full_match( From e67158b4baa0f9d32a2e819f71ddde822b977184 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 11 Nov 2024 12:44:06 +0100 Subject: [PATCH 201/489] feat: SP-1801 Update version, update changelog --- CHANGELOG.md | 4 ++++ src/scanoss/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1375f5ff..93d64178 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.18.0] - 2024-11-11 +### Added +- Add support for replace action when specifying a settings file + ## [1.17.5] - 2024-11-12 ### Fixed - Fix dependencies scan result structure diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 4c385f65..711333d4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.17.5' +__version__ = "1.18.0" From 8a2fbc330ef534fb77c59e6d3578b08aed515196 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 11 Nov 2024 13:52:12 +0100 Subject: [PATCH 202/489] feat: SP-1801 Add unit tests to workflow file --- .github/workflows/python-local-test.yml | 4 + .gitignore | 3 + tests/__init__.py | 0 tests/data/scanoss.json | 30 +++ tests/grpc-client-test.py | 55 +++--- tests/scancodedeps-test.py | 182 +++++++++--------- tests/scanpostprocessor-test.py | 97 ---------- .../{csvoutput-test.py => test_csvoutput.py} | 0 ...inspect-test.py => test_policy-inspect.py} | 9 +- tests/test_scanpostprocessor.py | 118 ++++++++++++ .../{winnowing-test.py => test_winnowing.py} | 0 11 files changed, 285 insertions(+), 213 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/data/scanoss.json delete mode 100644 tests/scanpostprocessor-test.py rename tests/{csvoutput-test.py => test_csvoutput.py} (100%) rename tests/{policy-inspect-test.py => test_policy-inspect.py} (98%) create mode 100644 tests/test_scanpostprocessor.py rename tests/{winnowing-test.py => test_winnowing.py} (100%) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 55931935..168ee56c 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -29,6 +29,10 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt + - name: Run Unit Tests + run: | + python -m unittest + - name: Build Local Package run: make dist diff --git a/.gitignore b/.gitignore index 361ada38..a0cb5a68 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ local-*.txt docs/build .devcontainer/devcontainer.json !.devcontainer/*.example.json + + +!tests/data/*.json diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/scanoss.json b/tests/data/scanoss.json new file mode 100644 index 00000000..b33ddcf1 --- /dev/null +++ b/tests/data/scanoss.json @@ -0,0 +1,30 @@ +{ + "bom": { + "include": [], + "remove": [ + { + "path": "scanoss_settings.py", + "purl": "pkg:github/scanoss/scanoss.py" + }, + { + "path": "test_file_path.go", + "purl": "pkg:github/scanoss/scanoss.py" + }, + { + "purl": "matching/purl" + } + ], + "replace": [ + { + "path": "full_match_test.py", + "purl": "pkg:github/scanoss/full_match_test.py", + "replace_with": "pkg:github/scanoss/full_match_replaced.py" + }, + { + "purl": "pkg:github/scanoss/only_purl_match.py", + "replace_with": "pkg:github/scanoss/only_purl_match_replaced.py" + } + ] + } +} + diff --git a/tests/grpc-client-test.py b/tests/grpc-client-test.py index fca2d3de..d9b2c609 100644 --- a/tests/grpc-client-test.py +++ b/tests/grpc-client-test.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import os import unittest @@ -32,17 +33,18 @@ class MyTestCase(unittest.TestCase): """ Unit test cases for GRPC comms """ - TEST_LOCAL = os.getenv("SCANOSS_TEST_LOCAL", 'True').lower() in ('true', '1', 't', 'yes', 'y') + + TEST_LOCAL = os.getenv('SCANOSS_TEST_LOCAL', 'True').lower() in ('true', '1', 't', 'yes', 'y') def test_grpc_dep_echo(self): """ Test the basic echo rpc call on the local server """ if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) echo_resp = grpc_client.deps_echo(f'testing dep echo ({server_type})') print(f'Echo Resp ({server_type}): {echo_resp}') @@ -53,23 +55,26 @@ def test_grpc_get_dependencies(self): Test getting dependencies from the local gRPC server """ sc_deps = ScancodeDeps(debug=True) - dep_file = "data/scancode-deps.json" + dep_file = 'data/scancode-deps.json' deps = sc_deps.produce_from_file(dep_file) print(f'Dependency JSON: {deps}') self.assertIsNotNone(deps) if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) resp = grpc_client.get_dependencies(deps) print(f'Resp ({server_type}): {resp}') self.assertIsNotNone(resp) - dep_files = resp.get("files") + dep_files = resp.get('files') if dep_files and len(dep_files) > 0: for dep_file in dep_files: - file = dep_file.pop("file", None) + file = dep_file.pop('file', None) print(f'File: {file} - {dep_file}') + +if __name__ == '__main__': + unittest.main() diff --git a/tests/scancodedeps-test.py b/tests/scancodedeps-test.py index 59811b85..fd754a8b 100644 --- a/tests/scancodedeps-test.py +++ b/tests/scancodedeps-test.py @@ -1,48 +1,50 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os import tempfile import unittest -from src.scanoss.scancodedeps import ScancodeDeps -from src.scanoss.scanossgrpc import ScanossGrpc -from src.scanoss.threadeddependencies import ThreadedDependencies, SCOPE +from scanoss.scancodedeps import ScancodeDeps +from scanoss.scanossgrpc import ScanossGrpc +from scanoss.threadeddependencies import SCOPE, ThreadedDependencies class MyTestCase(unittest.TestCase): """ Unit test cases for Scancode Dependency analysis """ - TEST_LOCAL = os.getenv("SCANOSS_TEST_LOCAL", 'False').lower() in ('true', '1', 't', 'yes', 'y') + + TEST_LOCAL = os.getenv('SCANOSS_TEST_LOCAL', 'False').lower() in ('true', '1', 't', 'yes', 'y') def test_deps_parse(self): """ Parse the saved scancode dependency data file """ sc_deps = ScancodeDeps(debug=True) - dep_file = "data/scancode-deps.json" + dep_file = 'data/scancode-deps.json' deps = sc_deps.produce_from_file(dep_file) print(f'Dependency JSON: {deps}') self.assertIsNotNone(deps) @@ -53,7 +55,7 @@ def test_scan_dir(self): """ sc_deps = ScancodeDeps(debug=True) - self.assertTrue(sc_deps.run_scan(what_to_scan=".")) + self.assertTrue(sc_deps.run_scan(what_to_scan='.')) deps = sc_deps.produce_from_file() sc_deps.remove_interim_file() print(f'Dependency JSON: {deps}') @@ -66,14 +68,14 @@ def test_threaded_scan_dir(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) - threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) - self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True)) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, '.', debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan='.', wait=True)) deps = threaded_deps.responses print(f'Dependency results ({server_type}): {deps}') self.assertIsNotNone(deps) @@ -85,24 +87,23 @@ def test_dep_scope_all(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) - threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) - self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True)) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, '.', debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan='.', wait=True)) deps = threaded_deps.responses - files = deps.get("files") - package_json_deps = files[0]["dependencies"] - requirements_txt_deps = files[1].get("dependencies", []) + files = deps.get('files') + package_json_deps = files[0]['dependencies'] + requirements_txt_deps = files[1].get('dependencies', []) print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') - self.assertEqual(len(package_json_deps),3) + self.assertEqual(len(package_json_deps), 3) self.assertEqual(len(requirements_txt_deps), 6) - def test_dep_scope_development(self): """ Run a dependency scan of the current directory, then parse those results @@ -110,24 +111,24 @@ def test_dep_scope_development(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) - threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) - self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope=SCOPE.DEVELOPMENT)) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, '.', debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan='.', wait=True, dep_scope=SCOPE.DEVELOPMENT)) deps = threaded_deps.responses - files = deps.get("files") - package_json_dev_deps = files[0]["dependencies"] - requirements_txt_dev_deps = files[1].get("dependencies", []) + files = deps.get('files') + package_json_dev_deps = files[0]['dependencies'] + requirements_txt_dev_deps = files[1].get('dependencies', []) print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_dev_deps}') print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_dev_deps}') - self.assertNotEquals(len(package_json_dev_deps),len(requirements_txt_dev_deps)) - self.assertEqual(len(package_json_dev_deps),1) + self.assertNotEquals(len(package_json_dev_deps), len(requirements_txt_dev_deps)) + self.assertEqual(len(package_json_dev_deps), 1) # devDependencies of package.json file: "@babel/core": ">0.2.0" - self.assertEqual(package_json_dev_deps[0]["component"], "@babel/core") + self.assertEqual(package_json_dev_deps[0]['component'], '@babel/core') def test_dep_scope_production(self): """ @@ -136,26 +137,26 @@ def test_dep_scope_production(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) - threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) - self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope=SCOPE.PRODUCTION)) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, '.', debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan='.', wait=True, dep_scope=SCOPE.PRODUCTION)) deps = threaded_deps.responses - files = deps.get("files") - package_json_deps = files[0]["dependencies"] - requirements_txt_deps = files[1].get("dependencies", []) + files = deps.get('files') + package_json_deps = files[0]['dependencies'] + requirements_txt_deps = files[1].get('dependencies', []) print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') - self.assertNotEquals(len(requirements_txt_deps),5) - self.assertEqual(len(package_json_deps),2) + self.assertNotEquals(len(requirements_txt_deps), 5) + self.assertEqual(len(package_json_deps), 2) - self.assertEqual(package_json_deps[0]["component"], "uuid") - self.assertEqual(package_json_deps[1]["component"], "xml-js") + self.assertEqual(package_json_deps[0]['component'], 'uuid') + self.assertEqual(package_json_deps[1]['component'], 'xml-js') def test_dep_scope_include(self): """ @@ -164,27 +165,27 @@ def test_dep_scope_include(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) - threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) - self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope_include='dependencies')) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, '.', debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan='.', wait=True, dep_scope_include='dependencies')) deps = threaded_deps.responses - files = deps.get("files") - package_json_deps = files[0]["dependencies"] - requirements_txt_deps = files[1].get("dependencies", []) + files = deps.get('files') + package_json_deps = files[0]['dependencies'] + requirements_txt_deps = files[1].get('dependencies', []) print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') # requirements.txt dependencies should be empty due to the filter 'dependencies' self.assertEqual(len(requirements_txt_deps), 0) - self.assertEqual(len(package_json_deps),2) + self.assertEqual(len(package_json_deps), 2) # Prod dependencies package.json file: "uuid" and "xml-js" - self.assertEqual(package_json_deps[0]["component"], "uuid") - self.assertEqual(package_json_deps[1]["component"], "xml-js") + self.assertEqual(package_json_deps[0]['component'], 'uuid') + self.assertEqual(package_json_deps[1]['component'], 'xml-js') def test_dep_scope_exclude(self): """ @@ -193,18 +194,18 @@ def test_dep_scope_exclude(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) - threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) - self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope_exclude='dependencies,install')) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, '.', debug=True, trace=True) + self.assertTrue(threaded_deps.run(what_to_scan='.', wait=True, dep_scope_exclude='dependencies,install')) deps = threaded_deps.responses - files = deps.get("files") - package_json_deps = files[0]["dependencies"] - requirements_txt_deps = files[1].get("dependencies", []) + files = deps.get('files') + package_json_deps = files[0]['dependencies'] + requirements_txt_deps = files[1].get('dependencies', []) print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') self.assertEqual(len(requirements_txt_deps), 0) @@ -213,7 +214,7 @@ def test_dep_scope_exclude(self): self.assertEqual(len(package_json_deps), 1) # Prod dependencies package.json file: "uuid" and "xml-js" - self.assertEqual(package_json_deps[0]["component"], "@babel/core") + self.assertEqual(package_json_deps[0]['component'], '@babel/core') def test_dep_scope_override(self): """ @@ -222,18 +223,22 @@ def test_dep_scope_override(self): # with open('scanoss-com.pem', 'rb') as f: # root_certs = f.read() if MyTestCase.TEST_LOCAL: - server_type = "local" + server_type = 'local' grpc_client = ScanossGrpc(debug=True, url='localhost:50051') else: - server_type = "remote" + server_type = 'remote' grpc_client = ScanossGrpc(debug=True) sc_deps = ScancodeDeps(debug=True) - threaded_deps = ThreadedDependencies(sc_deps, grpc_client, ".", debug=True, trace=True) - self.assertTrue(threaded_deps.run(what_to_scan=".", wait=True, dep_scope=SCOPE.PRODUCTION ,dep_scope_exclude='dependencies,install')) + threaded_deps = ThreadedDependencies(sc_deps, grpc_client, '.', debug=True, trace=True) + self.assertTrue( + threaded_deps.run( + what_to_scan='.', wait=True, dep_scope=SCOPE.PRODUCTION, dep_scope_exclude='dependencies,install' + ) + ) deps = threaded_deps.responses - files = deps.get("files") - package_json_deps = files[0]["dependencies"] - requirements_txt_deps = files[1].get("dependencies", []) + files = deps.get('files') + package_json_deps = files[0]['dependencies'] + requirements_txt_deps = files[1].get('dependencies', []) print(f'Dependency results for: ({files[0]["file"]}), dependencies: {package_json_deps}') print(f'Dependency results for: ({files[1]["file"]}), dependencies: {requirements_txt_deps}') self.assertEqual(len(requirements_txt_deps), 0) @@ -242,29 +247,30 @@ def test_dep_scope_override(self): self.assertEqual(len(package_json_deps), 1) # Prod dependencies package.json file: "uuid" and "xml-js" - self.assertEqual(package_json_deps[0]["component"], "@babel/core") + self.assertEqual(package_json_deps[0]['component'], '@babel/core') def test_dependency_scan(self): """ - Run a dependency scan of the current directory. Dependencies should be returned without scopes + Run a dependency scan of the current directory. Dependencies should be returned without scopes """ temp_dir = tempfile.gettempdir() - file_name = "dependency-result-output.json" + file_name = 'dependency-result-output.json' output_file = os.path.join(temp_dir, file_name) sc_deps = ScancodeDeps(debug=True, trace=True) - success = sc_deps.get_dependencies(what_to_scan=".",result_output=output_file) + success = sc_deps.get_dependencies(what_to_scan='.', result_output=output_file) self.assertTrue(success) with open(output_file, 'r') as result: # Parse the JSON data from the file dependencies = json.load(result) - files = dependencies.get("files") + files = dependencies.get('files') for file in files: - purls = file.get("purls") + purls = file.get('purls') contains_scope = any('scope' in purl for purl in purls) self.assertFalse(contains_scope) os.remove(output_file) + if __name__ == '__main__': unittest.main() diff --git a/tests/scanpostprocessor-test.py b/tests/scanpostprocessor-test.py deleted file mode 100644 index b462848c..00000000 --- a/tests/scanpostprocessor-test.py +++ /dev/null @@ -1,97 +0,0 @@ -""" - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -""" - -import unittest - -from scanoss.scanoss_settings import ScanossSettings -from scanoss.scanpostprocessor import ScanPostProcessor - - -class MyTestCase(unittest.TestCase): - """ - Unit test cases for Scan Post-Processing - """ - - scan_settings = ScanossSettings(filepath="tests/data/scanoss.json") - post_processor = ScanPostProcessor(scan_settings) - - def test_remove_files(self): - """ - Should remove component if matches path and purl - """ - - results = { - "scanoss_settings.py": [ - { - "purl": ["pkg:github/scanoss/scanoss.py"], - } - ], - } - processed_results = self.post_processor.load_results(results).post_process() - - self.assertEqual(len(processed_results), 0) - self.assertEqual(processed_results, {}) - - def test_remove_files_no_results(self): - """ - Should return empty dictionary when empty results are provided - """ - processed_results = self.post_processor.load_results({}).post_process() - - self.assertEqual(len(processed_results), 0) - self.assertEqual(processed_results, {}) - - def test_remove_files_path_match(self): - """ - Should remove component if matches path - """ - results = { - "test_file_path.go": [ - { - "purl": ["no/matching/purl"], - } - ], - } - processed_results = self.post_processor.load_results(results).post_process() - self.assertEqual(len(processed_results), 0) - self.assertEqual(processed_results, {}) - - def test_remove_files_purl_match(self): - """ - Should remove component if matches purl - """ - results = { - "no_matching_path.go": [ - { - "purl": ["matching/purl"], - } - ], - } - processed_results = self.post_processor.load_results(results).post_process() - self.assertEqual(len(processed_results), 0) - self.assertEqual(processed_results, {}) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/csvoutput-test.py b/tests/test_csvoutput.py similarity index 100% rename from tests/csvoutput-test.py rename to tests/test_csvoutput.py diff --git a/tests/policy-inspect-test.py b/tests/test_policy-inspect.py similarity index 98% rename from tests/policy-inspect-test.py rename to tests/test_policy-inspect.py index bd9e5cad..7fc41d6d 100644 --- a/tests/policy-inspect-test.py +++ b/tests/test_policy-inspect.py @@ -26,8 +26,8 @@ import re import unittest -from src.scanoss.inspection.copyleft import Copyleft -from src.scanoss.inspection.undeclared_component import UndeclaredComponent +from scanoss.inspection.copyleft import Copyleft +from scanoss.inspection.undeclared_component import UndeclaredComponent class MyTestCase(unittest.TestCase): @@ -240,4 +240,7 @@ def test_undeclared_policy_markdown(self): self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output)) self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), - re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) \ No newline at end of file + re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_scanpostprocessor.py b/tests/test_scanpostprocessor.py new file mode 100644 index 00000000..376be6af --- /dev/null +++ b/tests/test_scanpostprocessor.py @@ -0,0 +1,118 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import os +import unittest +from pathlib import Path + +from scanoss.scanoss_settings import ScanossSettings +from scanoss.scanpostprocessor import ScanPostProcessor + + +class MyTestCase(unittest.TestCase): + """ + Unit test cases for Scan Post-Processing + """ + + script_dir = os.path.dirname(os.path.abspath(__file__)) + scan_settings_path = Path(script_dir, 'data', 'scanoss.json').resolve() + scan_settings = ScanossSettings(filepath=scan_settings_path) + post_processor = ScanPostProcessor(scan_settings) + + def test_remove_files(self): + """ + Should remove component if matches path and purl + """ + + results = { + 'scanoss_settings.py': [ + { + 'purl': ['pkg:github/scanoss/scanoss.py'], + } + ], + } + processed_results = self.post_processor.load_results(results).post_process() + + self.assertEqual(len(processed_results), 0) + self.assertEqual(processed_results, {}) + + def test_remove_files_no_results(self): + """ + Should return empty dictionary when empty results are provided + """ + processed_results = self.post_processor.load_results({}).post_process() + + self.assertEqual(len(processed_results), 0) + self.assertEqual(processed_results, {}) + + def test_remove_files_purl_match(self): + """ + Should remove component if matches purl + """ + results = { + 'no_matching_path.go': [ + { + 'purl': ['matching/purl'], + } + ], + } + processed_results = self.post_processor.load_results(results).post_process() + self.assertEqual(len(processed_results), 0) + self.assertEqual(processed_results, {}) + + def test_replace_purls_full_match(self): + """ + Should replace purl if full match + """ + results = { + 'full_match_test.py': [ + { + 'purl': ['pkg:github/scanoss/full_match_test.py'], + } + ], + } + processed_results = self.post_processor.load_results(results).post_process() + self.assertEqual(len(processed_results), 1) + self.assertEqual( + processed_results['full_match_test.py'][0]['purl'], ['pkg:github/scanoss/full_match_replaced.py'] + ) + + def test_replace_purls_purl_match(self): + """Should replace purl if matches purl""" + results = { + 'only_purl_match.py': [ + { + 'purl': ['pkg:github/scanoss/only_purl_match.py'], + } + ], + } + processed_results = self.post_processor.load_results(results).post_process() + self.assertEqual(len(processed_results), 1) + self.assertEqual( + processed_results['only_purl_match.py'][0]['purl'], ['pkg:github/scanoss/only_purl_match_replaced.py'] + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/winnowing-test.py b/tests/test_winnowing.py similarity index 100% rename from tests/winnowing-test.py rename to tests/test_winnowing.py From 680a625946146e399470cc89f76c9c7a73a5aa27 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 11 Nov 2024 13:53:55 +0100 Subject: [PATCH 203/489] feat: SP-1801 Fix workflow --- .github/workflows/python-local-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 168ee56c..a14b8e04 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -29,10 +29,6 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt - - name: Run Unit Tests - run: | - python -m unittest - - name: Build Local Package run: make dist @@ -82,3 +78,7 @@ jobs: echo "Error: WFP test did not produce any results. Failing" exit 1 fi + + - name: Run Unit Tests + run: | + python -m unittest From 7caf119c7b7f962906e44ebdd2a19e6eac312735 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 14 Nov 2024 11:18:25 +0100 Subject: [PATCH 204/489] feat: SP-1801 Add replaced purls as context for scan, update replaced result with new component information if available --- src/scanoss/scanoss_settings.py | 53 ++++++++------- src/scanoss/scanpostprocessor.py | 113 ++++++++++++++++++++++++++----- 2 files changed, 126 insertions(+), 40 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index a6a9bfff..03901311 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -69,15 +69,15 @@ def load_json_file(self, filepath: str): json_file = Path(filepath).resolve() if not json_file.exists(): - self.print_stderr(f"Scan settings file not found: {filepath}") + self.print_stderr(f'Scan settings file not found: {filepath}') self.data = {} - with open(json_file, "r") as jsonfile: - self.print_debug(f"Loading scan settings from: {filepath}") + with open(json_file, 'r') as jsonfile: + self.print_debug(f'Loading scan settings from: {filepath}') try: self.data = json.load(jsonfile) except Exception as e: - self.print_stderr(f"ERROR: Problem parsing input JSON: {e}") + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return self def set_file_type(self, file_type: str): @@ -91,9 +91,7 @@ def set_file_type(self, file_type: str): """ self.settings_file_type = file_type if not self._is_valid_sbom_file: - raise Exception( - 'Invalid scan settings file, missing "components" or "bom")' - ) + raise Exception('Invalid scan settings file, missing "components" or "bom")') return self def set_scan_type(self, scan_type: str): @@ -111,7 +109,7 @@ def _is_valid_sbom_file(self): Returns: bool: True if the file is valid, False otherwise """ - if not self.data.get("components") or not self.data.get("bom"): + if not self.data.get('components') or not self.data.get('bom'): return False return True @@ -122,14 +120,14 @@ def _get_bom(self): dict: If using scanoss.json list: If using SBOM.json """ - if self.settings_file_type == "legacy": + if self.settings_file_type == 'legacy': if isinstance(self.data, list): return self.data - elif isinstance(self.data, dict) and self.data.get("components"): - return self.data.get("components") + elif isinstance(self.data, dict) and self.data.get('components'): + return self.data.get('components') else: return [] - return self.data.get("bom", {}) + return self.data.get('bom', {}) def get_bom_include(self) -> List[BomEntry]: """Get the list of components to include in the scan @@ -137,9 +135,9 @@ def get_bom_include(self) -> List[BomEntry]: Returns: list: List of components to include in the scan """ - if self.settings_file_type == "legacy": + if self.settings_file_type == 'legacy': return self._get_bom() - return self._get_bom().get("include", []) + return self._get_bom().get('include', []) def get_bom_remove(self) -> List[BomEntry]: """Get the list of components to remove from the scan @@ -147,9 +145,9 @@ def get_bom_remove(self) -> List[BomEntry]: Returns: list: List of components to remove from the scan """ - if self.settings_file_type == "legacy": + if self.settings_file_type == 'legacy': return self._get_bom() - return self._get_bom().get("remove", []) + return self._get_bom().get('remove', []) def get_bom_replace(self) -> List[BomEntry]: """Get the list of components to replace in the scan @@ -157,21 +155,21 @@ def get_bom_replace(self) -> List[BomEntry]: Returns: list: List of components to replace in the scan """ - if self.settings_file_type == "legacy": + if self.settings_file_type == 'legacy': return [] - return self._get_bom().get("replace", []) + return self._get_bom().get('replace', []) def get_sbom(self): """Get the SBOM to be sent to the SCANOSS API Returns: - dict: SBOM + dict: SBOM request payload """ if not self.data: return None return { - "scan_type": self.scan_type, - "assets": json.dumps(self._get_sbom_assets()), + 'scan_type': self.scan_type, + 'assets': json.dumps(self._get_sbom_assets()), } def _get_sbom_assets(self): @@ -180,8 +178,15 @@ def _get_sbom_assets(self): Returns: List: List of SBOM assets """ - if self.scan_type == "identify": - return self.normalize_bom_entries(self.get_bom_include()) + if self.scan_type == 'identify': + include_bom_entries = self.normalize_bom_entries(self.get_bom_include()) + replace_bom_entries = self.normalize_bom_entries(self.get_bom_replace()) + self.print_debug( + f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n" + f"From Include list: {[entry['purl'] for entry in include_bom_entries]} \n" + f"From Replace list: {[entry['purl'] for entry in replace_bom_entries]} \n" + ) + return include_bom_entries + replace_bom_entries return self.normalize_bom_entries(self.get_bom_remove()) @staticmethod @@ -198,7 +203,7 @@ def normalize_bom_entries(bom_entries) -> List[BomEntry]: for entry in bom_entries: normalized_bom_entries.append( { - "purl": entry.get("purl", ""), + 'purl': entry.get('purl', ''), } ) return normalized_bom_entries diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 208d0ce3..25890914 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -24,6 +24,9 @@ from typing import List, Tuple +from packageurl import PackageURL +from packageurl.contrib import purl2url + from .scanoss_settings import BomEntry, ScanossSettings from .scanossbase import ScanossBase @@ -50,6 +53,7 @@ def __init__( super().__init__(debug, trace, quiet) self.scan_settings = scan_settings self.results: dict = results + self.component_info_map: dict = {} def load_results(self, raw_results: dict): """Load the raw results @@ -58,19 +62,30 @@ def load_results(self, raw_results: dict): raw_results (dict): Raw scan results """ self.results = raw_results + self._load_component_info() return self + def _load_component_info(self): + """Create a map of component information from scan results for faster lookup""" + if not self.results: + return + for _, result in self.results.items(): + result = result[0] if isinstance(result, list) else result + purls = result.get('purl', []) + for purl in purls: + self.component_info_map[purl] = result + def post_process(self): """Post-process the scan results Returns: dict: Processed results """ - self.remove_dismissed_files() - self.replace_purls() + self._remove_dismissed_files() + self._replace_purls() return self.results - def remove_dismissed_files(self): + def _remove_dismissed_files(self): """Remove entries from the results based on files and/or purls specified in the SCANOSS settings file""" to_remove_entries = self.scan_settings.get_bom_remove() if not to_remove_entries: @@ -82,7 +97,7 @@ def remove_dismissed_files(self): if not self._should_remove_result(result_path, result, to_remove_entries) } - def replace_purls(self): + def _replace_purls(self): """Replace purls in the results based on the SCANOSS settings file""" to_replace_entries = self.scan_settings.get_bom_replace() if not to_replace_entries: @@ -90,9 +105,54 @@ def replace_purls(self): for result_path, result in self.results.items(): result = result[0] if isinstance(result, list) else result - should_replace, to_replace_with = self._should_replace_result(result_path, result, to_replace_entries) + should_replace, to_replace_with_purl = self._should_replace_result(result_path, result, to_replace_entries) if should_replace: - result['purl'] = [to_replace_with] + self.results[result_path] = self._update_replaced_result(result, to_replace_with_purl) + + def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> dict: + """ + Update the result with the new purl and component information if available, + otherwise removes the old component information + + Args: + result (dict): The result to update + to_replace_with_purl (str): The purl to replace with + + Returns: + dict: Updated result + """ + + if self.component_info_map.get(to_replace_with_purl): + result.update(self.component_info_map[to_replace_with_purl]) + else: + try: + new_component = PackageURL.from_string(to_replace_with_purl).to_dict() + new_component_url = purl2url.get_repo_url(to_replace_with_purl) + except Exception: + self.print_stderr( + f"Error while replacing: Invalid PURL '{to_replace_with_purl}' in settings file. Abort replacing." + ) + return result + + result['component'] = new_component.get('name') + result['url'] = new_component_url + result['vendor'] = new_component.get('namespace') + + result.pop('licenses', None) + result.pop('file', None) + result.pop('file_hash', None) + result.pop('file_url', None) + result.pop('latest', None) + result.pop('release_date', None) + result.pop('source_hash', None) + result.pop('url_hash', None) + result.pop('url_stats', None) + result.pop('url_stats', None) + result.pop('version', None) + + result['purl'] = [to_replace_with_purl] + + return result def _should_replace_result( self, result_path: str, result: dict, to_replace_entries: List[BomEntry] @@ -117,9 +177,6 @@ def _should_replace_result( if not to_replace_path and not to_replace_purl or not to_replace_with: continue - if to_replace_with in result_purls: - continue - if ( self._is_full_match(result_path, result_purls, to_replace_entry) or (not to_replace_path and to_replace_purl in result_purls) @@ -160,6 +217,36 @@ def _print_message( action: str, ) -> None: """Print a message about replacing or removing a result""" + message = ( + f"{self._get_match_type_message(result_path, result_purls, bom_entry, action)} \n" + f"Details:\n" + f" - PURLs: {', '.join(result_purls)}\n" + f" - Path: '{result_path}'\n" + ) + + if action == 'Replacing': + message += f" - {action} with '{bom_entry.get('replace_with')}'" + + self.print_debug(message) + + def _get_match_type_message( + self, + result_path: str, + result_purls: List[str], + bom_entry: BomEntry, + action: str, + ) -> str: + """Compose message based on match type + + Args: + result_path (str): Path of the scan result + result_purls (List[str]): Purls of the scan result + bom_entry (BomEntry): BOM entry to compare with + action (str): Post processing action being performed + + Returns: + str: The message to be printed + """ if bom_entry.get('path') and bom_entry.get('purl'): message = f"{action} '{result_path}'. Full match found." elif bom_entry.get('purl'): @@ -167,13 +254,7 @@ def _print_message( else: message = f"{action} '{result_path}'. Found path match." - self.print_debug( - f"{message}\n" - f"Details:\n" - f" - PURLs: {', '.join(result_purls)}\n" - f" - Path: '{result_path}'\n" - f" - {action} with: '{bom_entry.get('replace_with')}'\n" if action == 'Replacing' else '' - ) + return message def _is_full_match( self, From 9fd0502c2110a1cff037863aafdc7adc3a253008 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 14 Nov 2024 11:35:18 +0100 Subject: [PATCH 205/489] feat: SP-1801 Update requirements, update changelog, fix possible post processor attribute error if settings not set --- CHANGELOG.md | 3 +++ requirements.txt | 3 ++- setup.cfg | 2 ++ src/scanoss/scanner.py | 18 ++++++++++-------- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93d64178..b79b469b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Upcoming changes... ## [1.18.0] - 2024-11-11 +### Fixed +- Fixed post processor being accesed if not set ### Added - Add support for replace action when specifying a settings file +- Add replaced files as context to scan request ## [1.17.5] - 2024-11-12 ### Fixed diff --git a/requirements.txt b/requirements.txt index da90a28e..0b95ea31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ pypac urllib3 pyOpenSSL google-api-core -importlib_resources \ No newline at end of file +importlib_resources +packageurl-python \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index cdf17fcd..8de3fd05 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,8 @@ install_requires = pyOpenSSL google-api-core importlib_resources + packageurl-python + [options.extras_require] fast_winnowing = diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 7ad5932d..a55c2dcb 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -161,12 +161,13 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str if skip_extensions: # Append extra file extensions to skip self.skip_extensions.extend(skip_extensions) - if scan_settings: - self.scan_settings = scan_settings - self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) - self._maybe_set_api_sbom() + self.scan_settings = scan_settings + self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None + self._maybe_set_api_sbom() def _maybe_set_api_sbom(self): + if not self.scan_settings: + return sbom = self.scan_settings.get_sbom() if sbom: self.scanoss_api.set_sbom(sbom) @@ -521,11 +522,12 @@ def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> b success = False dep_responses = self.threaded_deps.responses - raw_scan_results = self._merge_scan_results( - scan_responses, dep_responses, file_map - ) + raw_scan_results = self._merge_scan_results(scan_responses, dep_responses, file_map) - results = self.post_processor.load_results(raw_scan_results).post_process() + if self.post_processor: + results = self.post_processor.load_results(raw_scan_results).post_process() + else: + results = raw_scan_results if self.output_format == 'plain': self.__log_result(json.dumps(results, indent=2, sort_keys=True)) From b8a180df0d6a83cf98c363575b1162e921cc9551 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 14 Nov 2024 11:40:37 +0100 Subject: [PATCH 206/489] feat: SP-1801 Replaced result should be a list instead of a dict --- src/scanoss/scanpostprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 25890914..d68ab52f 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -107,7 +107,7 @@ def _replace_purls(self): result = result[0] if isinstance(result, list) else result should_replace, to_replace_with_purl = self._should_replace_result(result_path, result, to_replace_entries) if should_replace: - self.results[result_path] = self._update_replaced_result(result, to_replace_with_purl) + self.results[result_path] = [self._update_replaced_result(result, to_replace_with_purl)] def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> dict: """ From d0a2693ba5db0ccb62c15e4405b4df70d50d67e3 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 14 Nov 2024 14:35:58 +0100 Subject: [PATCH 207/489] feat: SP-1801 Remove duplicate bom entries for sending as sbom payload --- src/scanoss/scanoss_settings.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 03901311..4065e357 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -179,8 +179,8 @@ def _get_sbom_assets(self): List: List of SBOM assets """ if self.scan_type == 'identify': - include_bom_entries = self.normalize_bom_entries(self.get_bom_include()) - replace_bom_entries = self.normalize_bom_entries(self.get_bom_replace()) + include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include())) + replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace())) self.print_debug( f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n" f"From Include list: {[entry['purl'] for entry in include_bom_entries]} \n" @@ -207,3 +207,22 @@ def normalize_bom_entries(bom_entries) -> List[BomEntry]: } ) return normalized_bom_entries + + @staticmethod + def _remove_duplicates(bom_entries: List[BomEntry]) -> List[BomEntry]: + """Remove duplicate BOM entries + + Args: + bom_entries (List[Dict]): List of BOM entries + + Returns: + List: List of unique BOM entries + """ + already_added = set() + unique_entries = [] + for entry in bom_entries: + entry_tuple = tuple(entry.items()) + if entry_tuple not in already_added: + already_added.add(entry_tuple) + unique_entries.append(entry) + return unique_entries From 793467a030ca728b47eb6d1c790fef125a0d0577 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 11 Nov 2024 13:52:12 +0100 Subject: [PATCH 208/489] feat: SP-1801 Add unit tests to workflow file --- .github/workflows/python-local-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index a14b8e04..e5d98367 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -29,6 +29,10 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt + - name: Run Unit Tests + run: | + python -m unittest + - name: Build Local Package run: make dist From be955119b593cbc2921ed0f0b3454b36708ed149 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 15 Nov 2024 11:58:00 +0100 Subject: [PATCH 209/489] feat: SP-1801 Fix local test workflow --- .github/workflows/python-local-test.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index e5d98367..480e2b04 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -5,10 +5,10 @@ on: workflow_dispatch: push: branches: - - 'main' + - "main" pull_request: branches: - - 'main' + - "main" permissions: contents: read @@ -22,17 +22,13 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: "3.10.x" - name: Install Dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - - name: Run Unit Tests - run: | - python -m unittest - - name: Build Local Package run: make dist @@ -82,7 +78,8 @@ jobs: echo "Error: WFP test did not produce any results. Failing" exit 1 fi - + - name: Run Unit Tests run: | python -m unittest + From 43f3a764b229fc2fdd6eb3d93fa093b2c3bf1cc6 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:40:28 -0300 Subject: [PATCH 210/489] feat:SP-1846 Adds sbom-format flag to inspect undeclared command * feat:SP-1846 Adds sbom-format flag to inspect undeclared command --- CHANGELOG.md | 8 +- CLIENT_HELP.md | 7 ++ src/scanoss/cli.py | 4 +- .../inspection/undeclared_component.py | 52 +++++++--- .../{test_csvoutput.py => test_csv_output.py} | 0 ...licy-inspect.py => test_policy_inspect.py} | 96 ++++++++++++++++++- ...ocessor.py => test_scan_post_processor.py} | 0 7 files changed, 149 insertions(+), 18 deletions(-) rename tests/{test_csvoutput.py => test_csv_output.py} (100%) rename tests/{test_policy-inspect.py => test_policy_inspect.py} (73%) rename tests/{test_scanpostprocessor.py => test_scan_post_processor.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b79b469b..7fe61f0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed post processor being accesed if not set ### Added -- Add support for replace action when specifying a settings file -- Add replaced files as context to scan request +- Added support for replace action when specifying a settings file +- Added replaced files as context to scan request +- Added sbom format flag to define status output for undeclared policy ## [1.17.5] - 2024-11-12 ### Fixed @@ -401,4 +402,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.17.2]: https://github.com/scanoss/scanoss.py/compare/v1.17.1...v1.17.2 [1.17.3]: https://github.com/scanoss/scanoss.py/compare/v1.17.2...v1.17.3 [1.17.4]: https://github.com/scanoss/scanoss.py/compare/v1.17.3...v1.17.4 -[1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 \ No newline at end of file +[1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 +[1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index a92212ac..3913303e 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -374,4 +374,11 @@ scanoss-py insp undeclared -i scan-results.json --status undeclared-status.md -- The following command can be used to inspect for undeclared components and save the results in Markdown format. ```bash scanoss-py insp undeclared -i scan-results.json --status undeclared-status.md --output undeclared.json --format md +``` + +#### Inspect for undeclared components and save results in Markdown format and show status output as sbom.json (legacy) +The following command can be used to inspect for undeclared components and save the results in Markdown format. +Default sbom-format 'settings' +```bash +scanoss-py insp undeclared -i scan-results.json --status undeclared-status.md --output undeclared.json --format md --sbom-format legacy ``` \ No newline at end of file diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 9eadcaca..8d2f7ad8 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -315,6 +315,8 @@ def setup_args() -> None: # Inspect Sub-command: inspect undeclared p_undeclared = p_inspect_sub.add_parser('undeclared', aliases=['un'],description="Inspect for undeclared components", help='Inspect for undeclared components') + p_undeclared.add_argument('--sbom-format',required=False ,choices=['legacy', 'settings'], + default="settings",help='Sbom format for status output') p_undeclared.set_defaults(func=inspect_undeclared) for p in [p_copyleft, p_undeclared]: @@ -858,7 +860,7 @@ def inspect_undeclared(parser, args): open(status_output, 'w').close() i_undeclared = UndeclaredComponent(debug=args.debug, trace=args.trace, quiet=args.quiet, filepath=args.input, format_type=args.format, - status=status_output, output=output) + status=status_output, output=output, sbom_format=args.sbom_format) status, _ = i_undeclared.run() sys.exit(status) diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index f18ff148..4e618fdc 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -32,7 +32,7 @@ class UndeclaredComponent(PolicyCheck): """ def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, - format_type: str = 'json', status: str = None, output: str = None): + format_type: str = 'json', status: str = None, output: str = None, sbom_format: str = 'settings'): """ Initialize the UndeclaredComponent class. @@ -43,6 +43,7 @@ def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, :param format_type: Output format ('json' or 'md') :param status: Path to save status output :param output: Path to save detailed output + :param sbom_format: Sbom format for status output (default 'settings') """ super().__init__(debug, trace, quiet, filepath, format_type, status, output, name='Undeclared Components Policy') @@ -50,6 +51,7 @@ def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, self.format = format self.output = output self.status = status + self.sbom_format = sbom_format def _get_undeclared_component(self, components: list)-> list or None: """ @@ -59,7 +61,7 @@ def _get_undeclared_component(self, components: list)-> list or None: :return: List of undeclared components """ if components is None: - self.print_stderr(f'WARNING: No components provided!') + self.print_debug(f'WARNING: No components provided!') return None undeclared_components = [] for component in components: @@ -78,9 +80,14 @@ def _get_summary(self, components: list) -> str: """ summary = f'{len(components)} undeclared component(s) were found.\n' if len(components) > 0: + if self.sbom_format == 'settings': + summary += (f'Add the following snippet into your `scanoss.json` file\n' + f'\n```json\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n```\n') + return summary + summary += (f'Add the following snippet into your `sbom.json` file\n' f'\n```json\n{json.dumps(self._generate_sbom_file(components), indent=2)}\n```\n') - return summary + return summary def _json(self, components: list) -> Dict[str, Any]: """ @@ -115,23 +122,46 @@ def _markdown(self, components: list) -> Dict[str,Any]: 'summary': self._get_summary(components), } - def _generate_sbom_file(self, components: list) -> dict: + def _get_unique_components(self, components: list) -> list: """ - Generate a list of PURLs for the SBOM file. + Generate a list of unique components. :param components: List of undeclared components - :return: SBOM Dictionary with components + :return: list of unique components """ - unique_components = {} if components is None: self.print_stderr(f'WARNING: No components provided!') - else: - for component in components: - unique_components[component['purl']] = { 'purl': component['purl'] } + return [] + + for component in components: + unique_components[component['purl']] = {'purl': component['purl']} + return list(unique_components.values()) + + def _generate_scanoss_file(self, components: list) -> dict: + """ + Generate a list of PURLs for the scanoss.json file. + + :param components: List of undeclared components + :return: scanoss.json Dictionary + """ + scanoss_settings = { + 'bom':{ + 'include': self._get_unique_components(components), + } + } + return scanoss_settings + + def _generate_sbom_file(self, components: list) -> dict: + """ + Generate a list of PURLs for the SBOM file. + + :param components: List of undeclared components + :return: SBOM Dictionary with components + """ sbom = { - 'components': list(unique_components.values()) + 'components': self._get_unique_components(components), } return sbom diff --git a/tests/test_csvoutput.py b/tests/test_csv_output.py similarity index 100% rename from tests/test_csvoutput.py rename to tests/test_csv_output.py diff --git a/tests/test_policy-inspect.py b/tests/test_policy_inspect.py similarity index 73% rename from tests/test_policy-inspect.py rename to tests/test_policy_inspect.py index 7fc41d6d..65b0f44b 100644 --- a/tests/test_policy-inspect.py +++ b/tests/test_policy_inspect.py @@ -165,7 +165,7 @@ def test_undeclared_policy(self): script_dir = os.path.dirname(os.path.abspath(__file__)) file_name = "result.json" input_file_name = os.path.join(script_dir,'data', file_name) - undeclared = UndeclaredComponent(filepath=input_file_name, format_type='json') + undeclared = UndeclaredComponent(filepath=input_file_name, format_type='json', sbom_format='legacy') status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] @@ -201,7 +201,7 @@ def test_undeclared_policy_markdown(self): script_dir = os.path.dirname(os.path.abspath(__file__)) file_name = "result.json" input_file_name = os.path.join(script_dir, 'data', file_name) - undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md') + undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md', sbom_format='legacy') status, results = undeclared.run() details = results['details'] summary = results['summary'] @@ -241,6 +241,96 @@ def test_undeclared_policy_markdown(self): '', expected_details_output)) self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) - + + """ + Undeclared component markdown scanoss summary output + """ + def test_undeclared_policy_markdown_scanoss_summary(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md') + status, results = undeclared.run() + details = results['details'] + summary = results['summary'] + expected_details_output = """ ### Undeclared components + | Component | Version | License | + | - | - | - | + | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | + | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | + | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | + | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | + | pkg:npm/%40emotion/react | 11.13.3 | MIT | """ + + expected_summary_output = """5 undeclared component(s) were found. + Add the following snippet into your `scanoss.json` file + + ```json + { + "bom": { + "include": [ + { + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + }, + { + "purl": "pkg:npm/%40electron/rebuild" + }, + { + "purl": "pkg:npm/%40emotion/react" + } + ] + } + } + ```""" + + print(summary) + self.assertEqual(status, 0) + self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', + '', expected_details_output)) + self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), + re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) + + """ + Undeclared component sbom summary output + """ + def test_undeclared_policy_scanoss_summary(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + undeclared = UndeclaredComponent(filepath=input_file_name) + status, results = undeclared.run() + details = json.loads(results['details']) + summary = results['summary'] + expected_summary_output = """5 undeclared component(s) were found. + Add the following snippet into your `scanoss.json` file + + ```json + { + "bom": { + "include": [ + { + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + }, + { + "purl": "pkg:npm/%40electron/rebuild" + }, + { + "purl": "pkg:npm/%40emotion/react" + } + ] + } + } + ```""" + self.assertEqual(status, 0) + self.assertEqual(len(details['components']), 5) + self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), + re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_scanpostprocessor.py b/tests/test_scan_post_processor.py similarity index 100% rename from tests/test_scanpostprocessor.py rename to tests/test_scan_post_processor.py From 3113e0fda1fe7841b4b950ae8da7b822d56b1618 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 15 Nov 2024 14:02:58 +0100 Subject: [PATCH 211/489] Add scanoss settings schema docs --- .gitignore | 2 +- docs/assets/scanoss-settings-schema.json | 132 +++++++++++++++++ docs/source/conf.py | 29 +++- docs/source/index.rst | 2 + docs/source/scanoss_settings_schema.rst | 175 +++++++++++++++++++++++ 5 files changed, 332 insertions(+), 8 deletions(-) create mode 100644 docs/assets/scanoss-settings-schema.json create mode 100644 docs/source/scanoss_settings_schema.rst diff --git a/.gitignore b/.gitignore index a0cb5a68..8f6bbb31 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,5 @@ docs/build .devcontainer/devcontainer.json !.devcontainer/*.example.json - !tests/data/*.json +!docs/assets/*.json diff --git a/docs/assets/scanoss-settings-schema.json b/docs/assets/scanoss-settings-schema.json new file mode 100644 index 00000000..b47beec6 --- /dev/null +++ b/docs/assets/scanoss-settings-schema.json @@ -0,0 +1,132 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Scanoss Settings", + "type": "object", + "properties": { + "self": { + "type": "object", + "description": "Description of the project under analysis", + "properties": { + "name": { + "type": "string", + "description": "Name of the project" + }, + "license": { + "type": "string", + "description": "License of the project" + }, + "description": { + "type": "string", + "description": "Description of the project" + } + } + }, + "bom": { + "type": "object", + "description": "BOM Rules: Set of rules that will be used to modify the BOM before and after the scan is completed", + "properties": { + "include": { + "type": "array", + "description": "Set of rules to be added as context when scanning. This list will be sent as payload to the API.", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path", + "examples": ["/path/to/file", "/path/to/another/file"], + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "purl": { + "type": "string", + "description": "Package URL to be used to match the component", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + }, + "comment": { + "type": "string", + "description": "Additional notes or comments" + } + }, + "uniqueItems": true, + "required": ["purl"] + } + }, + "remove": { + "type": "array", + "description": "Set of rules that will remove files from the results file after the scan is completed.", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path", + "examples": ["/path/to/file", "/path/to/another/file"] + }, + "purl": { + "type": "string", + "description": "Package URL", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + }, + "comment": { + "type": "string", + "description": "Additional notes or comments" + } + }, + "uniqueItems": true, + "required": ["purl"] + } + }, + "replace": { + "type": "array", + "description": "Set of rules that will replace components with the specified one after the scan is completed.", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path", + "examples": ["/path/to/file", "/path/to/another/file"] + }, + "purl": { + "type": "string", + "description": "Package URL to replace", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + }, + "comment": { + "type": "string", + "description": "Additional notes or comments" + }, + "license": { + "type": "string", + "description": "License of the component. Should be a valid SPDX license expression", + "examples": ["MIT", "Apache-2.0"] + }, + "replace_with": { + "type": "string", + "description": "Package URL to replace with", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + } + }, + "uniqueItems": true, + "required": ["purl", "replace_with"] + } + } + } + } + } +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 9d25efb0..b081422c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,23 +6,38 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'Documentation for scanoss-py' -copyright = '2024, Scan Open Source Solutions SL' -author = 'Scan Open Source Solutions SL' +import os +import shutil + +project = "Documentation for scanoss-py" +copyright = "2024, Scan Open Source Solutions SL" +author = "Scan Open Source Solutions SL" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [] -templates_path = ['_templates'] +templates_path = ["_templates"] exclude_patterns = [] +def setup(): + if not os.path.exists("_static"): + os.makedirs("_static") + + schema_path = os.path.join("..", "asets", "scanoss-settings-schema.json") + if os.path.exists(schema_path): + shutil.copy2(schema_path, "_static/scanoss-settings-schema.json") + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'furo' -html_logo = 'scanosslogo.png' -html_static_path = ['_static'] +html_theme = "furo" +html_logo = "scanosslogo.png" +html_static_path = ["_static"] + +html_context = { + "schema_url": "https://scanoss.readthedocs.io/en/latest/_static/scanoss-settings-schema.json" +} diff --git a/docs/source/index.rst b/docs/source/index.rst index 308f2ce9..3d855bd1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -46,6 +46,8 @@ To enable dependency scanning, an extra tool is required: scancode-toolkit. To install it run: ``pip3 install -r requirements-scancode.txt`` +.. include:: scanoss_settings_schema.rst + Commands and arguments ====================== diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst new file mode 100644 index 00000000..1b0bccb9 --- /dev/null +++ b/docs/source/scanoss_settings_schema.rst @@ -0,0 +1,175 @@ +Settings File +====================== + +SCANOSS provides a settings file to customize the scanning process. The settings file is a JSON file that contains project information and BOM (Bill of Materials) rules. It allows you to include, remove, or replace components in the BOM before and after scanning. + +The schema is available at: ``https://.readthedocs.io/en/latest/_static/schema.json`` + +Schema Overview +------------- + +The settings file consists of two main sections: + +Project Information +----------------- + +The ``self`` section contains basic information about the your project: + +.. code-block:: json + + { + "self": { + "name": "my-project", + "license": "MIT", + "description": "Project description" + } + } + +BOM Rules +--------- + +The ``bom`` section defines rules for modifying the BOM before and after scanning. It contains three main operations: + +1. Include Rules +~~~~~~~~~~~~~~ + +Rules for adding context when scanning. These rules will be sent to the SCANOSS API meaning they have more chance of being considered part of the resulting scan. + +.. code-block:: json + + { + "bom": { + "include": [ + { + "path": "/path/to/file", + "purl": "pkg:npm/vue@2.6.12", + "comment": "Optional comment" + } + ] + } + } + +2. Remove Rules +~~~~~~~~~~~~~ + +Rules for removing files from results after scanning. These rules will be applied to the results file after scanning. The post processing happens on the client side. + +.. code-block:: json + + { + "bom": { + "remove": [ + { + "path": "/path/to/file", + "purl": "pkg:npm/vue@2.6.12", + "comment": "Optional comment" + } + ] + } + } + +3. Replace Rules +~~~~~~~~~~~~~~ + +Rules for replacing components after scanning. These rules will be applied to the results file after scanning. The post processing happens on the client side. + +.. code-block:: json + + { + "bom": { + "replace": [ + { + "path": "/path/to/file", + "purl": "pkg:npm/vue@2.6.12", + "replace_with": "pkg:npm/vue@2.6.14", + "license": "MIT", + "comment": "Optional comment" + } + ] + } + } + +Important Notes +------------- + +Matching Rules +~~~~~~~~~~~~ + +1. **Full Match**: Requires both PATH and PURL to match. It means the rule will be applied ONLY to the specific file with the matching PURL and PATH. +2. **Partial Match**: Matches based on either: + - File path only (PURL is optional). It means the rule will be applied to all files with the matching path. + - PURL only (PATH is optional). It means the rule will be applied to all files with the matching PURL. + + +File Paths +~~~~~~~~~ + +- All paths should be specified relative to the scanned directory +- Use forward slashes (``/``) as path separators + +Given the following example directory structure: +.. code-block:: text + + project + ├── src + │ └── component.js + └── lib + └── utils.py + +- If the scanned directory is ``/project/src``, then: + - ``component.js`` is a valid path + - ``lib/utils.py`` is an invalid path and will not match any files +- If the scanned directory is ``/project``, then: + - ``src/component.js`` is a valid path + - ``lib/utils.py`` is a valid path + +Package URLs (PURLs) +~~~~~~~~~~~~~~~~~~ + +PURLs must follow the Package URL specification: + +- Format: ``pkg://@`` +- Examples: + - ``pkg:npm/vue@2.6.12`` + - ``pkg:golang/github.com/golang/go@1.17.3`` +- Must be valid and include all required components +- Version is strongly recommended but optional + +Example Configuration +------------------- + +Here's a complete example showing all sections: + +.. code-block:: json + + { + "self": { + "name": "example-project", + "license": "Apache-2.0", + "description": "Example project configuration" + }, + "bom": { + "include": [ + { + "path": "src/lib/component.js", + "purl": "pkg:npm/lodash@4.17.21", + "comment": "Include lodash dependency" + } + ], + "remove": [ + { + "purl": "pkg:npm/deprecated-pkg@1.0.0", + "comment": "Remove deprecated package" + } + ], + "replace": [ + { + "path": "src/utils/helper.js", + "purl": "pkg:npm/old-lib@1.0.0", + "replace_with": "pkg:npm/new-lib@2.0.0", + "license": "MIT", + "comment": "Upgrade to newer version" + } + ] + } + } \ No newline at end of file From 332aa821a125bb1d376cdcfb3f4b674d0f0e737b Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 15 Nov 2024 14:18:11 +0100 Subject: [PATCH 212/489] Update docs for proper indentation and formatting --- docs/source/scanoss_settings_schema.rst | 34 ++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index 1b0bccb9..28225f2b 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -6,14 +6,14 @@ SCANOSS provides a settings file to customize the scanning process. The settings The schema is available at: ``https://.readthedocs.io/en/latest/_static/schema.json`` Schema Overview -------------- +--------------- The settings file consists of two main sections: Project Information ------------------ +------------------- -The ``self`` section contains basic information about the your project: +The ``self`` section contains basic information about your project: .. code-block:: json @@ -31,7 +31,7 @@ BOM Rules The ``bom`` section defines rules for modifying the BOM before and after scanning. It contains three main operations: 1. Include Rules -~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~ Rules for adding context when scanning. These rules will be sent to the SCANOSS API meaning they have more chance of being considered part of the resulting scan. @@ -50,7 +50,7 @@ Rules for adding context when scanning. These rules will be sent to the SCANOSS } 2. Remove Rules -~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~ Rules for removing files from results after scanning. These rules will be applied to the results file after scanning. The post processing happens on the client side. @@ -69,7 +69,7 @@ Rules for removing files from results after scanning. These rules will be applie } 3. Replace Rules -~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~ Rules for replacing components after scanning. These rules will be applied to the results file after scanning. The post processing happens on the client side. @@ -90,31 +90,31 @@ Rules for replacing components after scanning. These rules will be applied to th } Important Notes -------------- +--------------- Matching Rules -~~~~~~~~~~~~ +~~~~~~~~~~~~~~ 1. **Full Match**: Requires both PATH and PURL to match. It means the rule will be applied ONLY to the specific file with the matching PURL and PATH. 2. **Partial Match**: Matches based on either: - File path only (PURL is optional). It means the rule will be applied to all files with the matching path. - PURL only (PATH is optional). It means the rule will be applied to all files with the matching PURL. - File Paths -~~~~~~~~~ +~~~~~~~~~~ - All paths should be specified relative to the scanned directory - Use forward slashes (``/``) as path separators Given the following example directory structure: + .. code-block:: text - project - ├── src - │ └── component.js - └── lib - └── utils.py + project/ + ├── src/ + │ └── component.js + └── lib/ + └── utils.py - If the scanned directory is ``/project/src``, then: - ``component.js`` is a valid path @@ -124,7 +124,7 @@ Given the following example directory structure: - ``lib/utils.py`` is a valid path Package URLs (PURLs) -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~ PURLs must follow the Package URL specification: @@ -136,7 +136,7 @@ PURLs must follow the Package URL specification: - Version is strongly recommended but optional Example Configuration -------------------- +--------------------- Here's a complete example showing all sections: From 98ee01cc671ebfc15281dca187bb1644a96bd208 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 15 Nov 2024 14:30:45 +0100 Subject: [PATCH 213/489] Update schema file linking --- .gitignore | 2 +- .../_static}/scanoss-settings-schema.json | 0 docs/source/conf.py | 15 --------------- docs/source/scanoss_settings_schema.rst | 2 +- 4 files changed, 2 insertions(+), 17 deletions(-) rename docs/{assets => source/_static}/scanoss-settings-schema.json (100%) diff --git a/.gitignore b/.gitignore index 8f6bbb31..8e474346 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,4 @@ docs/build !.devcontainer/*.example.json !tests/data/*.json -!docs/assets/*.json +!docs/source/_static/*.json diff --git a/docs/assets/scanoss-settings-schema.json b/docs/source/_static/scanoss-settings-schema.json similarity index 100% rename from docs/assets/scanoss-settings-schema.json rename to docs/source/_static/scanoss-settings-schema.json diff --git a/docs/source/conf.py b/docs/source/conf.py index b081422c..eab62a2e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,8 +6,6 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -import os -import shutil project = "Documentation for scanoss-py" copyright = "2024, Scan Open Source Solutions SL" @@ -22,22 +20,9 @@ exclude_patterns = [] -def setup(): - if not os.path.exists("_static"): - os.makedirs("_static") - - schema_path = os.path.join("..", "asets", "scanoss-settings-schema.json") - if os.path.exists(schema_path): - shutil.copy2(schema_path, "_static/scanoss-settings-schema.json") - - # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "furo" html_logo = "scanosslogo.png" html_static_path = ["_static"] - -html_context = { - "schema_url": "https://scanoss.readthedocs.io/en/latest/_static/scanoss-settings-schema.json" -} diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index 28225f2b..fed3b96a 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -3,7 +3,7 @@ Settings File SCANOSS provides a settings file to customize the scanning process. The settings file is a JSON file that contains project information and BOM (Bill of Materials) rules. It allows you to include, remove, or replace components in the BOM before and after scanning. -The schema is available at: ``https://.readthedocs.io/en/latest/_static/schema.json`` +The schema is available to download :download:`here ` Schema Overview --------------- From 5dff2e2ec25cd6fc8b75f3350987681c84d24b05 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 18 Nov 2024 16:00:48 +0100 Subject: [PATCH 214/489] feat: SP-1855 Use scanoss.json as default settings file --- src/scanoss/cli.py | 69 +++++++++++++++-------------- src/scanoss/scanner.py | 74 ++++++++++++++++++-------------- src/scanoss/scanoss_settings.py | 27 +++++++++--- src/scanoss/scanpostprocessor.py | 5 +++ src/scanoss/utils/__init__.py | 23 ++++++++++ src/scanoss/utils/file.py | 45 +++++++++++++++++++ 6 files changed, 170 insertions(+), 73 deletions(-) create mode 100644 src/scanoss/utils/__init__.py create mode 100644 src/scanoss/utils/file.py diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 8d2f7ad8..08c9db42 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -27,10 +27,13 @@ import sys import pypac +from scanoss.utils.file import validate_json_file + + from .inspection.copyleft import Copyleft from .inspection.undeclared_component import UndeclaredComponent from .threadeddependencies import SCOPE -from .scanoss_settings import ScanossSettings +from .scanoss_settings import DEFAULT_SCANOSS_JSON_FILE, ScanossSettings from .scancodedeps import ScancodeDeps from .scanner import Scanner from .scantype import ScanType @@ -113,6 +116,11 @@ def setup_args() -> None: type=str, help='Settings file to use for scanning (optional - default scanoss.json)', ) + p_scan.add_argument( + '--omit-settings-file', + action='store_true', + help='Omit default settings file (scanoss.json) if it exists', + ) # Sub-command: fingerprint @@ -547,39 +555,28 @@ def scan(parser, args): exit(1) if args.identify and args.settings: - print_stderr(f'ERROR: Cannot specify both --identify and --settings options.') + print_stderr('ERROR: Cannot specify both --identify and --settings options.') exit(1) - - def is_valid_file(file_path: str) -> bool: - if not os.path.exists(file_path) or not os.path.isfile(file_path): - print_stderr(f'Specified file does not exist or is not a file: {file_path}') - return False - if not Scanner.valid_json_file(file_path): - return False - return True - - scan_settings = ScanossSettings( - debug=args.debug, trace=args.trace, quiet=args.quiet - ) - - if args.identify: - if not is_valid_file(args.identify) or args.ignore: - exit(1) - scan_settings.load_json_file(args.identify).set_file_type( - 'legacy' - ).set_scan_type('identify') - elif args.ignore: - if not is_valid_file(args.ignore): - exit(1) - scan_settings.load_json_file(args.ignore).set_file_type('legacy').set_scan_type( - 'blacklist' - ) - elif args.settings: - if not is_valid_file(args.settings): + + if args.settings and args.omit_settings_file: + print_stderr('ERROR: Cannot specify both --settings and --omit-file-settings options.') + exit(1) + + scan_settings = None + + if args.omit_settings_file: + print_stderr('Omit settings file is set. Skipping...') + else: + scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) + try: + if args.identify: + scan_settings.load_json_file(args.identify).set_file_type('legacy').set_scan_type('identify') + elif args.ignore: + scan_settings.load_json_file(args.ignore).set_file_type('legacy').set_scan_type('blacklist') + else: + scan_settings.load_json_file(args.settings).set_file_type('new').set_scan_type('identify') + except Exception: exit(1) - scan_settings.load_json_file(args.settings).set_file_type('new').set_scan_type( - 'identify' - ) if args.dep: if not os.path.exists(args.dep) or not os.path.isfile(args.dep): @@ -587,11 +584,13 @@ def is_valid_file(file_path: str) -> bool: f'Specified --dep file does not exist or is not a file: {args.dep}' ) exit(1) - if not Scanner.valid_json_file(args.dep): # Make sure it's a valid JSON file + try: + validate_json_file(args.dep) + except Exception: exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: print_stderr( - f'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.' + 'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.' ) scan_output: str = None @@ -684,7 +683,7 @@ def is_valid_file(file_path: str) -> bool: skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, - scan_settings=scan_settings + scan_settings=scan_settings, ) if args.wfp: diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index a55c2dcb..8d919caf 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -32,6 +32,8 @@ from progress.spinner import Spinner from pypac.parser import PACFile +from scanoss.utils.file import validate_json_file + from .scanossapi import ScanossApi from .cyclonedx import CycloneDx from .spdxlite import SpdxLite @@ -96,18 +98,44 @@ class Scanner(ScanossBase): Handle the scanning of files, snippets and dependencies """ - def __init__(self, wfp: str = None, scan_output: str = None, output_format: str = 'plain', - debug: bool = False, trace: bool = False, quiet: bool = False, api_key: str = None, url: str = None, - flags: str = None, nb_threads: int = 5, - post_size: int = 32, timeout: int = 180, no_wfp_file: bool = False, - all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, - scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None, - obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None, - ca_cert: str = None, pac: PACFile = None, retry: int = 5, hpsm: bool = False, - skip_size: int = 0, skip_extensions=None, skip_folders=None, - strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None, - scan_settings: ScanossSettings = None - ): + def __init__( + self, + wfp: str = None, + scan_output: str = None, + output_format: str = 'plain', + debug: bool = False, + trace: bool = False, + quiet: bool = False, + api_key: str = None, + url: str = None, + flags: str = None, + nb_threads: int = 5, + post_size: int = 32, + timeout: int = 180, + no_wfp_file: bool = False, + all_extensions: bool = False, + all_folders: bool = False, + hidden_files_folders: bool = False, + scan_options: int = 7, + sc_timeout: int = 600, + sc_command: str = None, + grpc_url: str = None, + obfuscate: bool = False, + ignore_cert_errors: bool = False, + proxy: str = None, + grpc_proxy: str = None, + ca_cert: str = None, + pac: PACFile = None, + retry: int = 5, + hpsm: bool = False, + skip_size: int = 0, + skip_extensions=None, + skip_folders=None, + strip_hpsm_ids=None, + strip_snippet_ids=None, + skip_md5_ids=None, + scan_settings: ScanossSettings = None + ): """ Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning """ @@ -255,27 +283,7 @@ def __count_files_in_wfp_file(wfp_file: str): if WFP_FILE_START in line: count += 1 return count - - @staticmethod - def valid_json_file(json_file: str) -> bool: - """ - Validate if the specified file is indeed valid JSON - :param: str JSON file to load - :return bool True if valid, False otherwise - """ - if not json_file: - Scanner.print_stderr('ERROR: No JSON file provided to parse.') - return False - if not os.path.isfile(json_file): - Scanner.print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') - return False - try: - with open(json_file) as f: - json.load(f) - except Exception as e: - Scanner.print_stderr(f'Problem parsing JSON file "{json_file}": {e}') - return False - return True + @staticmethod def version_details() -> str: diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 4065e357..3b690cf4 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -26,8 +26,12 @@ from pathlib import Path from typing import List, TypedDict +from scanoss.utils.file import validate_json_file + from .scanossbase import ScanossBase +DEFAULT_SCANOSS_JSON_FILE = 'scanoss.json' + class BomEntry(TypedDict, total=False): purl: str @@ -61,23 +65,32 @@ def __init__( self.load_json_file(filepath) def load_json_file(self, filepath: str): - """Load the scan settings file + """Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. Args: filepath (str): Path to the SCANOSS settings file """ + + if not filepath: + filepath = DEFAULT_SCANOSS_JSON_FILE + json_file = Path(filepath).resolve() - if not json_file.exists(): - self.print_stderr(f'Scan settings file not found: {filepath}') - self.data = {} + if filepath == DEFAULT_SCANOSS_JSON_FILE and not json_file.exists(): + self.print_debug(f'Default settings file not found: {filepath}. Skipping...') + return self + + try: + validate_json_file(json_file) + except ValueError as e: + return self.print_stderr(f'ERROR: Problem with settings file. {e}') with open(json_file, 'r') as jsonfile: self.print_debug(f'Loading scan settings from: {filepath}') try: self.data = json.load(jsonfile) except Exception as e: - self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + self.print_stderr(f'ERROR: Problem parsing settings file. {e}') return self def set_file_type(self, file_type: str): @@ -226,3 +239,7 @@ def _remove_duplicates(bom_entries: List[BomEntry]) -> List[BomEntry]: already_added.add(entry_tuple) unique_entries.append(entry) return unique_entries + + def is_legacy(self): + """Check if the settings file is legacy""" + return self.settings_file_type == 'legacy' diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index d68ab52f..66ad0cac 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -81,6 +81,11 @@ def post_process(self): Returns: dict: Processed results """ + if self.scan_settings.is_legacy(): + self.print_stderr( + 'Legacy settings file detected. Post-processing is not supported for legacy settings file.' + ) + return self.results self._remove_dismissed_files() self._replace_purls() return self.results diff --git a/src/scanoss/utils/__init__.py b/src/scanoss/utils/__init__.py new file mode 100644 index 00000000..ebd5917f --- /dev/null +++ b/src/scanoss/utils/__init__.py @@ -0,0 +1,23 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py new file mode 100644 index 00000000..cb564ec6 --- /dev/null +++ b/src/scanoss/utils/file.py @@ -0,0 +1,45 @@ +import json +import os +import sys + + +def print_stderr(*args, **kwargs): + """ + Print the given message to STDERR + """ + print(*args, file=sys.stderr, **kwargs) + + +def is_valid_file(file_path: str) -> bool: + """Check if the specified file exists and is a file + + Args: + file_path (str): The file path + + Returns: + bool: True if valid, False otherwise + """ + if not os.path.exists(file_path) or not os.path.isfile(file_path): + print_stderr(f'Specified file does not exist or is not a file: {file_path}') + return False + return True + + +def validate_json_file(json_file_path: str) -> None: + """Validate if the specified file is indeed a valid JSON file + + Args: + json_file_path (str): The JSON file to validate + + Raises: + ValueError: If the JSON file is not valid + """ + if not json_file_path: + raise ValueError('No JSON file provided to parse.') + if not os.path.isfile(json_file_path): + raise ValueError(f'JSON file does not exist or is not a file: {json_file_path}') + try: + with open(json_file_path) as f: + json.load(f) + except Exception as e: + raise ValueError(f'Problem parsing JSON file "{json_file_path}": {e}') from e From dbe739e98bb035da007519008932f7b067c2dfb8 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 19 Nov 2024 10:05:37 +0100 Subject: [PATCH 215/489] feat: SP-1855 Update docs, change --omit for --skip --- docs/source/scanoss_settings_schema.rst | 21 ++++++++++++++++++++- src/scanoss/cli.py | 4 ++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index fed3b96a..77672547 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -172,4 +172,23 @@ Here's a complete example showing all sections: } ] } - } \ No newline at end of file + } + +Usage +----- + +You can pass the settings file path as an argument to the CLI + +.. code-block:: bash + + $ scanoss-py scan . --settings /path/to/settings.json + +If no settings file is provided, the default settings file will be used. +The default location for the settings file is ``scanoss.json`` in the current working directory. +If this file does not exist, settings will be omitted. + +You can also skip the default settings file: + +.. code-block:: bash + + $ scanoss-py scan . --skip-settings-file \ No newline at end of file diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 08c9db42..3bdd5118 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -117,7 +117,7 @@ def setup_args() -> None: help='Settings file to use for scanning (optional - default scanoss.json)', ) p_scan.add_argument( - '--omit-settings-file', + '--skip-settings-file', action='store_true', help='Omit default settings file (scanoss.json) if it exists', ) @@ -559,7 +559,7 @@ def scan(parser, args): exit(1) if args.settings and args.omit_settings_file: - print_stderr('ERROR: Cannot specify both --settings and --omit-file-settings options.') + print_stderr('ERROR: Cannot specify both --settings and --skip-file-settings options.') exit(1) scan_settings = None From 819b01c0367b2969bcd2d45d2c5591e6a4ca0a2f Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 19 Nov 2024 10:16:40 +0100 Subject: [PATCH 216/489] feat: SP-1855 Return tuple from validate json util --- src/scanoss/cli.py | 10 +++++----- src/scanoss/scanner.py | 2 -- src/scanoss/scanoss_settings.py | 7 +++---- src/scanoss/utils/file.py | 14 ++++++++------ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 3bdd5118..030ef28c 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -558,13 +558,13 @@ def scan(parser, args): print_stderr('ERROR: Cannot specify both --identify and --settings options.') exit(1) - if args.settings and args.omit_settings_file: + if args.settings and args.skip_settings_file: print_stderr('ERROR: Cannot specify both --settings and --skip-file-settings options.') exit(1) scan_settings = None - if args.omit_settings_file: + if args.skip_settings_file: print_stderr('Omit settings file is set. Skipping...') else: scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) @@ -584,9 +584,9 @@ def scan(parser, args): f'Specified --dep file does not exist or is not a file: {args.dep}' ) exit(1) - try: - validate_json_file(args.dep) - except Exception: + is_valid, error = validate_json_file(args.dep) + if not is_valid: + print_stderr(f'Error: Dependency file is not valid: {error}') exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: print_stderr( diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 8d919caf..83a193b7 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -32,8 +32,6 @@ from progress.spinner import Spinner from pypac.parser import PACFile -from scanoss.utils.file import validate_json_file - from .scanossapi import ScanossApi from .cyclonedx import CycloneDx from .spdxlite import SpdxLite diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 3b690cf4..21d2a23a 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -80,10 +80,9 @@ def load_json_file(self, filepath: str): self.print_debug(f'Default settings file not found: {filepath}. Skipping...') return self - try: - validate_json_file(json_file) - except ValueError as e: - return self.print_stderr(f'ERROR: Problem with settings file. {e}') + is_valid, error = validate_json_file(json_file) + if not is_valid: + return self.print_stderr(f'ERROR: Problem with settings file. {error}') with open(json_file, 'r') as jsonfile: self.print_debug(f'Loading scan settings from: {filepath}') diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py index cb564ec6..9e324dae 100644 --- a/src/scanoss/utils/file.py +++ b/src/scanoss/utils/file.py @@ -1,6 +1,7 @@ import json import os import sys +from typing import Tuple def print_stderr(*args, **kwargs): @@ -25,21 +26,22 @@ def is_valid_file(file_path: str) -> bool: return True -def validate_json_file(json_file_path: str) -> None: +def validate_json_file(json_file_path: str) -> Tuple[bool, str]: """Validate if the specified file is indeed a valid JSON file Args: json_file_path (str): The JSON file to validate - Raises: - ValueError: If the JSON file is not valid + Returns: + Tuple[bool, str]: A tuple containing a boolean indicating if the file is valid and a message """ if not json_file_path: - raise ValueError('No JSON file provided to parse.') + return False, 'No JSON file provided to parse.' if not os.path.isfile(json_file_path): - raise ValueError(f'JSON file does not exist or is not a file: {json_file_path}') + return False, f'JSON file does not exist or is not a file: {json_file_path}' try: with open(json_file_path) as f: json.load(f) + return True, '' except Exception as e: - raise ValueError(f'Problem parsing JSON file "{json_file_path}": {e}') from e + return False, f'Problem parsing JSON file "{json_file_path}": {e}' From c86278acf03112e1e0549784fc6948847369f435 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 19 Nov 2024 10:42:32 +0100 Subject: [PATCH 217/489] feat: SP-1855 Add custom errors --- src/scanoss/cli.py | 11 ++++++----- src/scanoss/scanoss_settings.py | 22 +++++++++++----------- src/scanoss/utils/file.py | 24 ++++++++++++++++-------- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 030ef28c..937d8715 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -33,7 +33,7 @@ from .inspection.copyleft import Copyleft from .inspection.undeclared_component import UndeclaredComponent from .threadeddependencies import SCOPE -from .scanoss_settings import DEFAULT_SCANOSS_JSON_FILE, ScanossSettings +from .scanoss_settings import ScanossSettings, ScanossSettingsError from .scancodedeps import ScancodeDeps from .scanner import Scanner from .scantype import ScanType @@ -575,7 +575,8 @@ def scan(parser, args): scan_settings.load_json_file(args.ignore).set_file_type('legacy').set_scan_type('blacklist') else: scan_settings.load_json_file(args.settings).set_file_type('new').set_scan_type('identify') - except Exception: + except ScanossSettingsError as e: + print_stderr(f'Error: {e}') exit(1) if args.dep: @@ -584,9 +585,9 @@ def scan(parser, args): f'Specified --dep file does not exist or is not a file: {args.dep}' ) exit(1) - is_valid, error = validate_json_file(args.dep) - if not is_valid: - print_stderr(f'Error: Dependency file is not valid: {error}') + result = validate_json_file(args.dep) + if not result.is_valid: + print_stderr(f'Error: Dependency file is not valid: {result.error}') exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: print_stderr( diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 21d2a23a..d53f35ff 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -38,6 +38,10 @@ class BomEntry(TypedDict, total=False): path: str +class ScanossSettingsError(Exception): + pass + + class ScanossSettings(ScanossBase): """Handles the loading and parsing of the SCANOSS settings file""" @@ -64,7 +68,7 @@ def __init__( if filepath: self.load_json_file(filepath) - def load_json_file(self, filepath: str): + def load_json_file(self, filepath: str) -> 'ScanossSettings': """Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. Args: @@ -77,19 +81,15 @@ def load_json_file(self, filepath: str): json_file = Path(filepath).resolve() if filepath == DEFAULT_SCANOSS_JSON_FILE and not json_file.exists(): - self.print_debug(f'Default settings file not found: {filepath}. Skipping...') + self.print_debug(f"Default settings file '{filepath}' not found. Skipping...") return self - is_valid, error = validate_json_file(json_file) - if not is_valid: - return self.print_stderr(f'ERROR: Problem with settings file. {error}') + result = validate_json_file(json_file) + if not result.is_valid: + raise ScanossSettingsError(f'Problem with settings file. {result.error}') - with open(json_file, 'r') as jsonfile: - self.print_debug(f'Loading scan settings from: {filepath}') - try: - self.data = json.load(jsonfile) - except Exception as e: - self.print_stderr(f'ERROR: Problem parsing settings file. {e}') + self.data = result.data + self.print_debug(f'Loading scan settings from: {filepath}') return self def set_file_type(self, file_type: str): diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py index 9e324dae..cd68bdac 100644 --- a/src/scanoss/utils/file.py +++ b/src/scanoss/utils/file.py @@ -1,7 +1,8 @@ import json import os import sys -from typing import Tuple +from dataclasses import dataclass +from typing import Optional def print_stderr(*args, **kwargs): @@ -26,7 +27,14 @@ def is_valid_file(file_path: str) -> bool: return True -def validate_json_file(json_file_path: str) -> Tuple[bool, str]: +@dataclass +class JsonValidation: + is_valid: bool + data: Optional[dict] = None + error: Optional[str] = None + + +def validate_json_file(json_file_path: str) -> JsonValidation: """Validate if the specified file is indeed a valid JSON file Args: @@ -36,12 +44,12 @@ def validate_json_file(json_file_path: str) -> Tuple[bool, str]: Tuple[bool, str]: A tuple containing a boolean indicating if the file is valid and a message """ if not json_file_path: - return False, 'No JSON file provided to parse.' + return JsonValidation(is_valid=False, error='No JSON file specified') if not os.path.isfile(json_file_path): - return False, f'JSON file does not exist or is not a file: {json_file_path}' + return JsonValidation(is_valid=False, error=f'File not found: {json_file_path}') try: with open(json_file_path) as f: - json.load(f) - return True, '' - except Exception as e: - return False, f'Problem parsing JSON file "{json_file_path}": {e}' + data = json.load(f) + return JsonValidation(is_valid=True, data=data) + except json.JSONDecodeError as e: + return JsonValidation(is_valid=False, error=f'Problem parsing JSON file: "{json_file_path}": {e}') From aebc770291c93692d0e5f1fae4606a94ddcd9718 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Tue, 19 Nov 2024 10:58:05 +0000 Subject: [PATCH 218/489] option and status updates --- src/scanoss/cli.py | 72 +++++++++------------------------ src/scanoss/scanoss_settings.py | 62 +++++++++++++--------------- 2 files changed, 48 insertions(+), 86 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 937d8715..52b9be8b 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -112,17 +112,15 @@ def setup_args() -> None: p_scan.add_argument('--dep-scope-inc', '-dsi', type=str,help='Include dependencies with declared scopes') p_scan.add_argument('--dep-scope-exc', '-dse', type=str, help='Exclude dependencies with declared scopes') p_scan.add_argument( - '--settings', + '--settings', '-st', type=str, help='Settings file to use for scanning (optional - default scanoss.json)', ) p_scan.add_argument( - '--skip-settings-file', - action='store_true', - help='Omit default settings file (scanoss.json) if it exists', + '--skip-settings-file', '-stf', action='store_true', + help='Skip default settings file (scanoss.json) if it exists', ) - # Sub-command: fingerprint p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'], description=f'Fingerprint the given source base: {__version__}', @@ -537,13 +535,7 @@ def scan(parser, args): args: Namespace Parsed arguments """ - if ( - not args.scan_dir - and not args.wfp - and not args.stdin - and not args.dep - and not args.files - ): + if not args.scan_dir and not args.wfp and not args.stdin and not args.dep and not args.files: print_stderr( 'Please specify a file/folder, files (--files), fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)' ) @@ -553,20 +545,15 @@ def scan(parser, args): print_stderr('Please specify one of --proxy or --pac, not both') parser.parse_args([args.subparser, '-h']) exit(1) - if args.identify and args.settings: print_stderr('ERROR: Cannot specify both --identify and --settings options.') exit(1) - if args.settings and args.skip_settings_file: print_stderr('ERROR: Cannot specify both --settings and --skip-file-settings options.') exit(1) - + # Figure out which settings (if any) to load before processing scan_settings = None - - if args.skip_settings_file: - print_stderr('Omit settings file is set. Skipping...') - else: + if not args.skip_settings_file: scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) try: if args.identify: @@ -578,21 +565,16 @@ def scan(parser, args): except ScanossSettingsError as e: print_stderr(f'Error: {e}') exit(1) - if args.dep: if not os.path.exists(args.dep) or not os.path.isfile(args.dep): - print_stderr( - f'Specified --dep file does not exist or is not a file: {args.dep}' - ) + print_stderr(f'Specified --dep file does not exist or is not a file: {args.dep}') exit(1) result = validate_json_file(args.dep) if not result.is_valid: print_stderr(f'Error: Dependency file is not valid: {result.error}') exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: - print_stderr( - 'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.' - ) + print_stderr('Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') scan_output: str = None if args.output: @@ -601,6 +583,8 @@ def scan(parser, args): output_format = args.format if args.format else 'plain' flags = args.flags if args.flags else None if args.debug and not args.quiet: + if args.skip_settings_file: + print_stderr('Skipping Settings file...') if args.all_extensions: print_stderr("Scanning all file extensions/types...") if args.all_folders: @@ -631,17 +615,11 @@ def scan(parser, args): print_stderr(f'Using flags {flags}...') elif not args.quiet: if args.timeout < 5: - print_stderr( - f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.' - ) + print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') if args.retry < 0: - print_stderr( - f'POST retry (--retry) too small: {args.retry}. Reverting to default.' - ) + print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.') - if not os.access( - os.getcwd(), os.W_OK - ): # Make sure the current directory is writable. If not disable saving WFP + if not os.access(os.getcwd(), os.W_OK): # Make sure the current directory is writable. If not disable saving WFP print_stderr(f'Warning: Current directory is not writable: {os.getcwd()}') args.no_wfp_output = True if args.ca_cert and not os.path.exists(args.ca_cert): @@ -651,11 +629,8 @@ def scan(parser, args): scan_options = get_scan_options(args) # Figure out what scanning options we have scanner = Scanner( - debug=args.debug, - trace=args.trace, - quiet=args.quiet, - api_key=args.key, - url=args.apiurl, + debug=args.debug, trace=args.trace, quiet=args.quiet, + api_key=args.key, url=args.apiurl, scan_output=scan_output, output_format=output_format, flags=flags, @@ -686,17 +661,12 @@ def scan(parser, args): strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, ) - if args.wfp: if not scanner.is_file_or_snippet_scan(): - print_stderr( - f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})' - ) + print_stderr(f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})') exit(1) if scanner.is_dependency_scan() and not args.dep: - print_stderr( - f'Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)' - ) + print_stderr(f'Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)') exit(1) scanner.scan_wfp_with_options(args.wfp, args.dep) elif args.stdin: @@ -710,9 +680,7 @@ def scan(parser, args): exit(1) elif args.scan_dir: if not os.path.exists(args.scan_dir): - print_stderr( - f'Error: File or folder specified does not exist: {args.scan_dir}.' - ) + print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') exit(1) if os.path.isdir(args.scan_dir): if not scanner.scan_folder_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map, @@ -723,9 +691,7 @@ def scan(parser, args): args.dep_scope, args.dep_scope_inc, args.dep_scope_exc): exit(1) else: - print_stderr( - f'Error: Path specified is neither a file or a folder: {args.scan_dir}.' - ) + print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') exit(1) elif args.dep: if not args.dependencies_only: diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index d53f35ff..3e79208b 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -43,8 +43,9 @@ class ScanossSettingsError(Exception): class ScanossSettings(ScanossBase): - """Handles the loading and parsing of the SCANOSS settings file""" - + """ + Handles the loading and parsing of the SCANOSS settings file + """ def __init__( self, debug: bool = False, @@ -64,37 +65,34 @@ def __init__( self.data = {} self.settings_file_type = None self.scan_type = None - if filepath: self.load_json_file(filepath) def load_json_file(self, filepath: str) -> 'ScanossSettings': - """Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. + """ + Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. Args: filepath (str): Path to the SCANOSS settings file """ - if not filepath: filepath = DEFAULT_SCANOSS_JSON_FILE - json_file = Path(filepath).resolve() if filepath == DEFAULT_SCANOSS_JSON_FILE and not json_file.exists(): - self.print_debug(f"Default settings file '{filepath}' not found. Skipping...") + self.print_debug(f'Default settings file "{filepath}" not found. Skipping...') return self + self.print_msg(f'Loading settings file {filepath}...') result = validate_json_file(json_file) if not result.is_valid: raise ScanossSettingsError(f'Problem with settings file. {result.error}') - self.data = result.data - self.print_debug(f'Loading scan settings from: {filepath}') return self def set_file_type(self, file_type: str): - """Set the file type in order to support both legacy SBOM.json and new scanoss.json files - + """ + Set the file type in order to support both legacy SBOM.json and new scanoss.json files Args: file_type (str): 'legacy' or 'new' @@ -107,8 +105,8 @@ def set_file_type(self, file_type: str): return self def set_scan_type(self, scan_type: str): - """Set the scan type to support legacy SBOM.json files - + """ + Set the scan type to support legacy SBOM.json files Args: scan_type (str): 'identify' or 'exclude' """ @@ -116,8 +114,8 @@ def set_scan_type(self, scan_type: str): return self def _is_valid_sbom_file(self): - """Check if the scan settings file is valid - + """ + Check if the scan settings file is valid Returns: bool: True if the file is valid, False otherwise """ @@ -126,8 +124,8 @@ def _is_valid_sbom_file(self): return True def _get_bom(self): - """Get the Billing of Materials from the settings file - + """ + Get the Billing of Materials from the settings file Returns: dict: If using scanoss.json list: If using SBOM.json @@ -142,8 +140,8 @@ def _get_bom(self): return self.data.get('bom', {}) def get_bom_include(self) -> List[BomEntry]: - """Get the list of components to include in the scan - + """ + Get the list of components to include in the scan Returns: list: List of components to include in the scan """ @@ -152,8 +150,8 @@ def get_bom_include(self) -> List[BomEntry]: return self._get_bom().get('include', []) def get_bom_remove(self) -> List[BomEntry]: - """Get the list of components to remove from the scan - + """ + Get the list of components to remove from the scan Returns: list: List of components to remove from the scan """ @@ -162,8 +160,8 @@ def get_bom_remove(self) -> List[BomEntry]: return self._get_bom().get('remove', []) def get_bom_replace(self) -> List[BomEntry]: - """Get the list of components to replace in the scan - + """ + Get the list of components to replace in the scan Returns: list: List of components to replace in the scan """ @@ -172,8 +170,8 @@ def get_bom_replace(self) -> List[BomEntry]: return self._get_bom().get('replace', []) def get_sbom(self): - """Get the SBOM to be sent to the SCANOSS API - + """ + Get the SBOM to be sent to the SCANOSS API Returns: dict: SBOM request payload """ @@ -185,8 +183,8 @@ def get_sbom(self): } def _get_sbom_assets(self): - """Get the SBOM assets - + """ + Get the SBOM assets Returns: List: List of SBOM assets """ @@ -203,11 +201,10 @@ def _get_sbom_assets(self): @staticmethod def normalize_bom_entries(bom_entries) -> List[BomEntry]: - """Normalize the BOM entries - + """ + Normalize the BOM entries Args: bom_entries (List[Dict]): List of BOM entries - Returns: List: Normalized BOM entries """ @@ -222,11 +219,10 @@ def normalize_bom_entries(bom_entries) -> List[BomEntry]: @staticmethod def _remove_duplicates(bom_entries: List[BomEntry]) -> List[BomEntry]: - """Remove duplicate BOM entries - + """ + Remove duplicate BOM entries Args: bom_entries (List[Dict]): List of BOM entries - Returns: List: List of unique BOM entries """ From 8f7b16379514d942c63b179384404ba55e4e9261 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 19 Nov 2024 10:08:29 -0300 Subject: [PATCH 219/489] chore:SP-1861 Adds component field in CycloneDX output --- CHANGELOG.md | 7 ++++++- src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fe61f0b..ab008b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.18.1] - 2024-11-19 +### Added +- Added 'component' field in CycloneDX output + ## [1.18.0] - 2024-11-11 ### Fixed - Fixed post processor being accesed if not set @@ -403,4 +407,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.17.3]: https://github.com/scanoss/scanoss.py/compare/v1.17.2...v1.17.3 [1.17.4]: https://github.com/scanoss/scanoss.py/compare/v1.17.3...v1.17.4 [1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 -[1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 \ No newline at end of file +[1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 +[1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.1 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 711333d4..64991e78 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.18.0" +__version__ = "1.18.1" diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 91212707..d275ea06 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -199,6 +199,11 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: } ] }, + 'component': { + 'type': 'application', + 'name': 'NOASSERTION', + 'version': 'NOASSERTION' + }, 'components': [], 'vulnerabilities': [] } From 6c7b3db98e8d4dcfc3b738b0f528a19c5bdf9270 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 19 Nov 2024 10:56:22 +0100 Subject: [PATCH 220/489] feat: SP-1856 Modify scanoss settings schema --- CHANGELOG.md | 11 +++- .../_static/scanoss-settings-schema.json | 54 +++++++++++++++++++ src/scanoss/__init__.py | 2 +- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab008b3e..5b4000f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.19.0] - 2024-11-20 +### Fixed +- Check if legacy sbom file before post processing +### Added +- Use scanoss.json as default settings file if no argument is supplied +- Add —skip-settings-file flag +- Update scanoss settings schema to allow skipping specific folders, files, and extensions + ## [1.18.1] - 2024-11-19 ### Added - Added 'component' field in CycloneDX output @@ -408,4 +416,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.17.4]: https://github.com/scanoss/scanoss.py/compare/v1.17.3...v1.17.4 [1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 [1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 -[1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.1 \ No newline at end of file +[1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.1 +[1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.0...v1.19.0 \ No newline at end of file diff --git a/docs/source/_static/scanoss-settings-schema.json b/docs/source/_static/scanoss-settings-schema.json index b47beec6..8e511cf5 100644 --- a/docs/source/_static/scanoss-settings-schema.json +++ b/docs/source/_static/scanoss-settings-schema.json @@ -21,6 +21,60 @@ } } }, + "settings": { + "type": "object", + "description": "Scan settings and other configurations", + "properties": { + "skip": { + "type": "object", + "description": "Set of rules to skip files from the scan", + "properties": { + "folders": { + "type": "array", + "description": "List of folders to skip from the scan. These should be relative to the scan root", + "items": { + "type": "string", + "examples": ["/path/to/folder", "/path/to/another/folder"] + }, + "uniqueItems": true + }, + "files": { + "type": "array", + "description": "List of files to skip from the scan. These can be either relative file paths or just file names", + "items": { + "type": "string", + "examples": ["/path/to/include.h", "include.h"] + }, + "uniqueItems": true + }, + "extensions": { + "type": "array", + "description": "List of file extensions to skip from the scan", + "items": { + "type": "string", + "examples": [".h", ".c", ".cpp"] + }, + "uniqueItems": true + }, + "sizes": { + "type": "object", + "description": "Set of rules to skip files based on their size", + "properties": { + "min": { + "type": "integer", + "description": "Minimum size of the file in bytes" + }, + "max": { + "type": "integer", + "description": "Maximum size of the file in bytes" + } + }, + "required": ["max"] + } + } + } + } + }, "bom": { "type": "object", "description": "BOM Rules: Set of rules that will be used to modify the BOM before and after the scan is completed", diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 64991e78..74f26e03 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.18.1" +__version__ = "1.19.0" From d827f66ad9578238f301df990e9bf36fe837efb5 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 19 Nov 2024 16:37:50 +0100 Subject: [PATCH 221/489] feat: SP-1856 Initialize scan filter class --- src/scanoss/scan_filter.py | 256 ++++++++++++++++++++++++++++++++ src/scanoss/scanner.py | 98 ++++++------ src/scanoss/scanoss_settings.py | 20 +++ 3 files changed, 325 insertions(+), 49 deletions(-) create mode 100644 src/scanoss/scan_filter.py diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py new file mode 100644 index 00000000..f7ff285a --- /dev/null +++ b/src/scanoss/scan_filter.py @@ -0,0 +1,256 @@ +from pathlib import Path + +import pathspec + +from scanoss.scanossbase import ScanossBase + +DEFAULT_SKIPPED_FILES = { + 'gradlew', + 'gradlew.bat', + 'mvnw', + 'mvnw.cmd', + 'gradle-wrapper.jar', + 'maven-wrapper.jar', + 'thumbs.db', + 'babel.config.js', + 'license.txt', + 'license.md', + 'copying.lib', + 'makefile', +} + +DEFAULT_SKIPPED_DIRS = { # Folders to skip + 'nbproject', + 'nbbuild', + 'nbdist', + '__pycache__', + 'venv', + '_yardoc', + 'eggs', + 'wheels', + 'htmlcov', + '__pypackages__', +} +DEFAULT_SKIPPED_DIR_EXT = { # Folder endings to skip + '.egg-info' +} +DEFAULT_SKIPPED_EXT = [ # File extensions to skip + '.1', + '.2', + '.3', + '.4', + '.5', + '.6', + '.7', + '.8', + '.9', + '.ac', + '.adoc', + '.am', + '.asciidoc', + '.bmp', + '.build', + '.cfg', + '.chm', + '.class', + '.cmake', + '.cnf', + '.conf', + '.config', + '.contributors', + '.copying', + '.crt', + '.csproj', + '.css', + '.csv', + '.dat', + '.data', + '.doc', + '.docx', + '.dtd', + '.dts', + '.iws', + '.c9', + '.c9revisions', + '.dtsi', + '.dump', + '.eot', + '.eps', + '.geojson', + '.gdoc', + '.gif', + '.glif', + '.gmo', + '.gradle', + '.guess', + '.hex', + '.htm', + '.html', + '.ico', + '.iml', + '.in', + '.inc', + '.info', + '.ini', + '.ipynb', + '.jpeg', + '.jpg', + '.json', + '.jsonld', + '.lock', + '.log', + '.m4', + '.map', + '.markdown', + '.md', + '.md5', + '.meta', + '.mk', + '.mxml', + '.o', + '.otf', + '.out', + '.pbtxt', + '.pdf', + '.pem', + '.phtml', + '.plist', + '.png', + '.po', + '.ppt', + '.prefs', + '.properties', + '.pyc', + '.qdoc', + '.result', + '.rgb', + '.rst', + '.scss', + '.sha', + '.sha1', + '.sha2', + '.sha256', + '.sln', + '.spec', + '.sql', + '.sub', + '.svg', + '.svn-base', + '.tab', + '.template', + '.test', + '.tex', + '.tiff', + '.toml', + '.ttf', + '.txt', + '.utf-8', + '.vim', + '.wav', + '.woff', + '.woff2', + '.xht', + '.xhtml', + '.xls', + '.xlsx', + '.xml', + '.xpm', + '.xsd', + '.xul', + '.yaml', + '.yml', + '.wfp', + '.editorconfig', + '.dotcover', + '.pid', + '.lcov', + '.egg', + '.manifest', + '.cache', + '.coverage', + '.cover', + '.gem', + '.lst', + '.pickle', + '.pdb', + '.gml', + '.pot', + '.plt', + # File endings + '-doc', + 'changelog', + 'config', + 'copying', + 'license', + 'authors', + 'news', + 'licenses', + 'notice', + 'readme', + 'swiftdoc', + 'texidoc', + 'todo', + 'version', + 'ignore', + 'manifest', + 'sqlite', + 'sqlite3', +] + + +class ScanFilter(ScanossBase): + """ + Filter for determining which files to process during scanning. + Handles both inclusion and exclusion rules based on file paths, extensions, and sizes. + """ + + def __init__( + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + scan_root: Path = None, + settings: dict = None, + ): + """ + Initialize filter with settings from ScanossSettings. + + Args: + settings (ScanossSettings): Settings instance containing scan configuration + """ + super().__init__(debug, trace, quiet) + + self.scan_root = scan_root + + skip = settings.get('skip', {}) + skip_patterns = [] + + skip_patterns.extend(f'**/*{ext}' for ext in DEFAULT_SKIPPED_EXT) + skip_patterns.extend(DEFAULT_SKIPPED_FILES) + skip_patterns.extend(f'**/{dir}/**' for dir in DEFAULT_SKIPPED_DIRS) + skip_patterns.extend(f'**/*{ext}/**' for ext in DEFAULT_SKIPPED_DIR_EXT) + + skip_patterns_from_settings = [] + + # Add scan root to patterns, to support relative paths + for pattern in skip.get('patterns', []): + pattern_path = Path(scan_root, pattern) + skip_patterns_from_settings.append(str(pattern_path)) + skip_patterns.extend(skip.get('patterns', [])) + + self.skip_spec = pathspec.PathSpec.from_lines('gitwildmatch', skip_patterns) + self.min_size = skip.get('sizes', {}).get('min', 0) + self.max_size = skip.get('sizes', {}).get('max', float('inf')) + + def should_process(self, path: Path) -> bool: + if self.skip_spec.match_file(path): + self.print_debug(f'Skipping {path} {"folder" if path.is_dir() else "file"} due to pattern match') + return False + + if path.is_file(): + filesize = path.stat().st_size + if not (self.min_size <= filesize <= self.max_size): + self.print_debug(f'Skipping {path} due to size') + return False + + return True diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 83a193b7..5e0f0ba6 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -23,6 +23,7 @@ """ import json import os +from pathlib import Path import sys import datetime from typing import Any, Dict, List, Optional @@ -396,9 +397,9 @@ def scan_folder(self, scan_dir: str) -> bool: """ success = True if not scan_dir: - raise Exception(f"ERROR: Please specify a folder to scan") + raise Exception('ERROR: Please specify a folder to scan') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): - raise Exception(f"ERROR: Specified folder does not exist or is not a folder: {scan_dir}") + raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir) + 1 self.print_msg(f'Searching {scan_dir} for files to fingerprint...') @@ -413,57 +414,56 @@ def scan_folder(self, scan_dir: str) -> bool: file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False + for root, dirs, files in os.walk(scan_dir): self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break - dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories - filtered_files = self.__filter_files(files) # Strip out unwanted files - self.print_debug(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}') - for file in filtered_files: # Cycle through each filtered file - path = os.path.join(root, file) - f_size = 0 - try: - f_size = os.stat(path).st_size - except Exception as e: - self.print_trace( - f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink - # Ignore broken links and empty files or if a user-specified size limit is supplied - if f_size > 0 and (self.skip_size <= 0 or f_size > self.skip_size): - self.print_trace(f'Fingerprinting {path}...') - if spinner: - spinner.next() - wfp = self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) - if wfp is None or wfp == '': - self.print_debug(f'No WFP returned for {path}. Skipping.') - continue - if save_wfps_for_print: - wfp_list.append(wfp) - file_count += 1 - if self.threaded_scan: - wfp_size = len(wfp.encode("utf-8")) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue - if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - scan_block += wfp - scan_size = len(scan_block.encode("utf-8")) - wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue - if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do - scan_started = True - if not self.threaded_scan.run(wait=False): - self.print_stderr( - f'Warning: Some errors encounted while scanning. Results might be incomplete.') - success = False + + dirs[:] = [d for d in dirs if self.scan_settings.should_process(Path(root, d))] + + for file in files: + path = Path(root, file) + if not self.scan_settings.should_process(path): + self.print_debug(f'Skipping filtered file: {path}') + continue + + self.print_trace(f'Fingerprinting {path}...') + if spinner: + spinner.next() + wfp = self.winnowing.wfp_for_file(str(path), self.__strip_dir(scan_dir, scan_dir_len, str(path))) + if wfp is None or wfp == '': + self.print_debug(f'No WFP returned for {path}. Skipping.') + continue + if save_wfps_for_print: + wfp_list.append(wfp) + file_count += 1 + if self.threaded_scan: + wfp_size = len(wfp.encode('utf-8')) + # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + scan_block += wfp + scan_size = len(scan_block.encode('utf-8')) + wfp_file_count += 1 + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do + scan_started = True + if not self.threaded_scan.run(wait=False): + self.print_stderr( + 'Warning: Some errors encounted while scanning. Results might be incomplete.' + ) + success = False + # End for loop if self.threaded_scan and scan_block != '': self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted @@ -480,7 +480,7 @@ def scan_folder(self, scan_dir: str) -> bool: if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: - Scanner.print_stderr(f'Warning: No files found to scan in folder: {scan_dir}') + self.print_stderr(f'Warning: No files found to scan in folder: {scan_dir}') return success def __run_scan_threaded(self, scan_started: bool, file_count: int) -> bool: diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 3e79208b..24659676 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import List, TypedDict +from scanoss.scan_filter import ScanFilter from scanoss.utils.file import validate_json_file from .scanossbase import ScanossBase @@ -51,6 +52,7 @@ def __init__( debug: bool = False, trace: bool = False, quiet: bool = False, + scan_root: Path = None, filepath: str = None, ): """ @@ -62,9 +64,12 @@ def __init__( """ super().__init__(debug, trace, quiet) + self.scan_root = scan_root self.data = {} self.settings_file_type = None self.scan_type = None + self.filter = None + if filepath: self.load_json_file(filepath) @@ -88,6 +93,14 @@ def load_json_file(self, filepath: str) -> 'ScanossSettings': if not result.is_valid: raise ScanossSettingsError(f'Problem with settings file. {result.error}') self.data = result.data + self.filter = ScanFilter( + debug=self.debug, + quiet=self.quiet, + trace=self.trace, + settings=self.data.get('settings', {}), + scan_root=self.scan_root, + ) + self.print_debug(f'Loading scan settings from: {filepath}') return self def set_file_type(self, file_type: str): @@ -238,3 +251,10 @@ def _remove_duplicates(bom_entries: List[BomEntry]) -> List[BomEntry]: def is_legacy(self): """Check if the settings file is legacy""" return self.settings_file_type == 'legacy' + + def should_process(self, path: Path) -> bool: + """Check if file should be processed based on settings""" + if not self.filter: + return True + + return self.filter.should_process(path) From a86361b2578c16f05af935df778aff3a55071823 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 19 Nov 2024 18:46:57 +0100 Subject: [PATCH 222/489] feat: SP-1856 Refactor to pattern matching --- .../_static/scanoss-settings-schema.json | 34 +++----- src/scanoss/scan_filter.py | 80 +++++++++++++------ src/scanoss/scanner.py | 54 ++++++------- src/scanoss/scanoss_settings.py | 11 +-- 4 files changed, 92 insertions(+), 87 deletions(-) diff --git a/docs/source/_static/scanoss-settings-schema.json b/docs/source/_static/scanoss-settings-schema.json index 8e511cf5..575af6bb 100644 --- a/docs/source/_static/scanoss-settings-schema.json +++ b/docs/source/_static/scanoss-settings-schema.json @@ -29,30 +29,22 @@ "type": "object", "description": "Set of rules to skip files from the scan", "properties": { - "folders": { + "patterns": { "type": "array", - "description": "List of folders to skip from the scan. These should be relative to the scan root", + "description": "List of glob patterns to skip files", "items": { "type": "string", - "examples": ["/path/to/folder", "/path/to/another/folder"] - }, - "uniqueItems": true - }, - "files": { - "type": "array", - "description": "List of files to skip from the scan. These can be either relative file paths or just file names", - "items": { - "type": "string", - "examples": ["/path/to/include.h", "include.h"] - }, - "uniqueItems": true - }, - "extensions": { - "type": "array", - "description": "List of file extensions to skip from the scan", - "items": { - "type": "string", - "examples": [".h", ".c", ".cpp"] + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + + "path/to/file.c", + "path/to/another/file.py", + + "**/*.ts", + "**/*.json" + ] }, "uniqueItems": true }, diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index f7ff285a..96b36f71 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -1,6 +1,7 @@ -from pathlib import Path +import os +from typing import List, Set, Tuple -import pathspec +from pathspec import PathSpec from scanoss.scanossbase import ScanossBase @@ -209,7 +210,6 @@ def __init__( debug: bool = False, trace: bool = False, quiet: bool = False, - scan_root: Path = None, settings: dict = None, ): """ @@ -220,37 +220,65 @@ def __init__( """ super().__init__(debug, trace, quiet) - self.scan_root = scan_root - skip = settings.get('skip', {}) skip_patterns = [] - skip_patterns.extend(f'**/*{ext}' for ext in DEFAULT_SKIPPED_EXT) skip_patterns.extend(DEFAULT_SKIPPED_FILES) - skip_patterns.extend(f'**/{dir}/**' for dir in DEFAULT_SKIPPED_DIRS) - skip_patterns.extend(f'**/*{ext}/**' for ext in DEFAULT_SKIPPED_DIR_EXT) - - skip_patterns_from_settings = [] - - # Add scan root to patterns, to support relative paths - for pattern in skip.get('patterns', []): - pattern_path = Path(scan_root, pattern) - skip_patterns_from_settings.append(str(pattern_path)) + skip_patterns.extend(f'{dir}/' for dir in DEFAULT_SKIPPED_DIRS) + skip_patterns.extend(f'*{ext}' for ext in DEFAULT_SKIPPED_EXT) + skip_patterns.extend(f'*{ext}/' for ext in DEFAULT_SKIPPED_DIR_EXT) skip_patterns.extend(skip.get('patterns', [])) - self.skip_spec = pathspec.PathSpec.from_lines('gitwildmatch', skip_patterns) + self.skip_patterns = skip_patterns self.min_size = skip.get('sizes', {}).get('min', 0) self.max_size = skip.get('sizes', {}).get('max', float('inf')) - def should_process(self, path: Path) -> bool: - if self.skip_spec.match_file(path): - self.print_debug(f'Skipping {path} {"folder" if path.is_dir() else "file"} due to pattern match') - return False + def get_filtered_files(self, root: str) -> List[str]: + """Get a list of files to scan based on the filter settings. + + Args: + root (str): Root directory to scan + + Returns: + list[str]: List of files to scan + """ + files = self._walk_with_ignore(root) + return files + + def _walk_with_ignore(self, scan_root: str) -> List[str]: + files = [] + root = os.path.abspath(scan_root) + + path_spec, dir_patterns = self._create_skip_path_matchers() + + for dirpath, dirnames, filenames in os.walk(root): + rel_path = os.path.relpath(dirpath, root) + + # Return early if the entire directory should be skipped + if any(rel_path.startswith(p) for p in dir_patterns): + self.print_debug(f'Skipping directory: {rel_path}') + dirnames.clear() + continue + + for filename in filenames: + file_rel_path = os.path.join(rel_path, filename) + file_path = os.path.join(dirpath, filename) + file_size = os.path.getsize(file_path) + + if file_size < self.min_size or file_size > self.max_size: + self.print_debug(f'Skipping file: {file_rel_path} (size: {file_size})') + continue + if path_spec.match_file(file_rel_path): + self.print_debug(f'Skipping file: {file_rel_path}') + continue + else: + files.append(file_rel_path) + + return files + + def _create_skip_path_matchers(self) -> Tuple[PathSpec, Set[str]]: + dir_patterns = {p.rstrip('/') for p in self.skip_patterns if p.endswith('/')} - if path.is_file(): - filesize = path.stat().st_size - if not (self.min_size <= filesize <= self.max_size): - self.print_debug(f'Skipping {path} due to size') - return False + path_spec = PathSpec.from_lines('gitwildmatch', self.skip_patterns) - return True + return path_spec, dir_patterns diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 5e0f0ba6..c12e6da4 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -414,39 +414,31 @@ def scan_folder(self, scan_dir: str) -> bool: file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False - - for root, dirs, files in os.walk(scan_dir): - self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') + + to_scan_files = self.scan_settings.get_filtered_files(scan_dir) + + for to_scan_file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break - - dirs[:] = [d for d in dirs if self.scan_settings.should_process(Path(root, d))] - - for file in files: - path = Path(root, file) - if not self.scan_settings.should_process(path): - self.print_debug(f'Skipping filtered file: {path}') - continue - - self.print_trace(f'Fingerprinting {path}...') - if spinner: - spinner.next() - wfp = self.winnowing.wfp_for_file(str(path), self.__strip_dir(scan_dir, scan_dir_len, str(path))) - if wfp is None or wfp == '': - self.print_debug(f'No WFP returned for {path}. Skipping.') - continue - if save_wfps_for_print: - wfp_list.append(wfp) - file_count += 1 - if self.threaded_scan: - wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue - if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 + if spinner: + spinner.next() + abs_path = Path(scan_dir, to_scan_file).resolve() + wfp = self.winnowing.wfp_for_file(str(abs_path), to_scan_file) + if wfp is None or wfp == '': + self.print_debug(f'No WFP returned for {to_scan_file}. Skipping.') + continue + if save_wfps_for_print: + wfp_list.append(wfp) + file_count += 1 + if self.threaded_scan: + wfp_size = len(wfp.encode('utf-8')) + # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 scan_block += wfp scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 @@ -463,7 +455,7 @@ def scan_folder(self, scan_dir: str) -> bool: 'Warning: Some errors encounted while scanning. Results might be incomplete.' ) success = False - + # End for loop if self.threaded_scan and scan_block != '': self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 24659676..cb463e82 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -52,7 +52,6 @@ def __init__( debug: bool = False, trace: bool = False, quiet: bool = False, - scan_root: Path = None, filepath: str = None, ): """ @@ -64,7 +63,6 @@ def __init__( """ super().__init__(debug, trace, quiet) - self.scan_root = scan_root self.data = {} self.settings_file_type = None self.scan_type = None @@ -98,7 +96,6 @@ def load_json_file(self, filepath: str) -> 'ScanossSettings': quiet=self.quiet, trace=self.trace, settings=self.data.get('settings', {}), - scan_root=self.scan_root, ) self.print_debug(f'Loading scan settings from: {filepath}') return self @@ -252,9 +249,5 @@ def is_legacy(self): """Check if the settings file is legacy""" return self.settings_file_type == 'legacy' - def should_process(self, path: Path) -> bool: - """Check if file should be processed based on settings""" - if not self.filter: - return True - - return self.filter.should_process(path) + def get_filtered_files(self, scan_root: str) -> List[str]: + return self.filter.get_filtered_files(scan_root) From 58bd5c10cbab108809517af7a9645fa08ed8bc94 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 19 Nov 2024 19:08:00 +0100 Subject: [PATCH 223/489] feat: SP-1856 Fix scan folder method --- src/scanoss/scanner.py | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index c12e6da4..2ac1f955 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -416,11 +416,11 @@ def scan_folder(self, scan_dir: str) -> bool: scan_started = False to_scan_files = self.scan_settings.get_filtered_files(scan_dir) - - for to_scan_file in to_scan_files: + for to_scan_file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break + self.print_trace(f'Fingerprinting {to_scan_file}...') if spinner: spinner.next() abs_path = Path(scan_dir, to_scan_file).resolve() @@ -432,30 +432,28 @@ def scan_folder(self, scan_dir: str) -> bool: wfp_list.append(wfp) file_count += 1 if self.threaded_scan: - wfp_size = len(wfp.encode('utf-8')) + wfp_size = len(wfp.encode("utf-8")) # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - scan_block += wfp - scan_size = len(scan_block.encode('utf-8')) - wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue - if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 scan_block = '' wfp_file_count = 0 - if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do - scan_started = True - if not self.threaded_scan.run(wait=False): - self.print_stderr( - 'Warning: Some errors encounted while scanning. Results might be incomplete.' - ) - success = False - + scan_block += wfp + scan_size = len(scan_block.encode("utf-8")) + wfp_file_count += 1 + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do + scan_started = True + if not self.threaded_scan.run(wait=False): + self.print_stderr('Warning: Some errors encounted while scanning. Results might be incomplete.') + success = False + # End for loop if self.threaded_scan and scan_block != '': self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted @@ -472,7 +470,7 @@ def scan_folder(self, scan_dir: str) -> bool: if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: - self.print_stderr(f'Warning: No files found to scan in folder: {scan_dir}') + Scanner.print_stderr(f'Warning: No files found to scan in folder: {scan_dir}') return success def __run_scan_threaded(self, scan_started: bool, file_count: int) -> bool: From 12c80b88b379a9a651cf7b071df6d838b2cf1c45 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 20 Nov 2024 14:06:04 +0100 Subject: [PATCH 224/489] feat: SP-1856 Fix skip dir --- docs/source/scanoss_settings_schema.rst | 128 ++++++++++++++++++++++++ src/scanoss/scan_filter.py | 21 ++-- src/scanoss/scanner.py | 1 - 3 files changed, 137 insertions(+), 13 deletions(-) diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index 77672547..85521a3f 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -25,6 +25,134 @@ The ``self`` section contains basic information about your project: } } + +Settings +======== +The ``settings`` object allows you to configure various aspects of the scanning process. Currently, it provides control over which files should be skipped during scanning through the ``skip`` property. + +Skip Configuration +------------------ +The ``skip`` object lets you define rules for excluding files from being scanned. This can be useful for improving scan performance and avoiding unnecessary processing of certain files. + +Properties +~~~~~~~~~~ + +skip.patterns +^^^^^^^^^^^^^ +A list of patterns that determine which files should be skipped during scanning. The patterns follow the same format as ``.gitignore`` files. For more information, see the `gitignore patterns documentation `_. + +:Type: Array of strings +:Required: No +:Example: + .. code-block:: json + + { + "settings": { + "skip": { + "patterns": [ + "*.log", + "!important.log", + "temp/", + "debug[0-9]*.txt", + "src/client/specific-file.js", + "src/nested/folder/" + ] + } + } + +Pattern Format Rules +'''''''''''''''''''' +* Patterns are matched **relative to the scan root directory** +* A trailing slash indicates a directory (e.g., ``path/`` matches only directories) +* An asterisk ``*`` matches anything except a slash +* Two asterisks ``**`` match zero or more directories (e.g., ``path/**/folder`` matches ``path/to``, ``path/to/folder``, ``path/to/folder/b``) +* Range notations like ``[0-9]`` match any character in the range +* Question mark ``?`` matches any single character except a slash + + +Examples with Explanations +'''''''''''''''''''''''''' +.. code-block:: none + + # Match all .txt files + *.txt + + # Match all .log files except important.log + *.log + !important.log + + # Match all files in the build directory + build/ + + # Match all .pdf files in docs directory and its subdirectories + docs/**/*.pdf + + # Match files like test1.js, test2.js, etc. + test[0-9].js + +skip.sizes +^^^^^^^^^^ +Rules for skipping files based on their size. + +:Type: Object +:Required: No +:Properties: + * ``min`` (integer): Minimum file size in bytes + * ``max`` (integer): Maximum file size in bytes (Required) +:Example: + .. code-block:: json + + { + "settings": { + "skip": { + "sizes": { + "min": 100, + "max": 1000000 + } + } + } + } + +Complete Example +------------------- +Here's a comprehensive example combining pattern and size-based skipping: + +.. code-block:: json + + { + "settings": { + "skip": { + "patterns": [ + "# Node.js dependencies", + "node_modules/", + + "# Build outputs", + "dist/", + "build/", + + "# Logs except important ones", + "*.log", + "!important.log", + + "# Temporary files", + "temp/", + "*.tmp", + + "# Debug files with numbers", + "debug[0-9]*.txt", + + "# All test files in any directory", + "**/*test.js" + ], + "sizes": { + "min": 512, + "max": 5242880 + } + } + } + } + + BOM Rules --------- diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index 96b36f71..b7a7c4e9 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -1,5 +1,5 @@ import os -from typing import List, Set, Tuple +from typing import List from pathspec import PathSpec @@ -230,6 +230,7 @@ def __init__( skip_patterns.extend(skip.get('patterns', [])) self.skip_patterns = skip_patterns + self.path_spec = PathSpec.from_lines('gitwildmatch', self.skip_patterns) self.min_size = skip.get('sizes', {}).get('min', 0) self.max_size = skip.get('sizes', {}).get('max', float('inf')) @@ -249,13 +250,11 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: files = [] root = os.path.abspath(scan_root) - path_spec, dir_patterns = self._create_skip_path_matchers() - for dirpath, dirnames, filenames in os.walk(root): rel_path = os.path.relpath(dirpath, root) - # Return early if the entire directory should be skipped - if any(rel_path.startswith(p) for p in dir_patterns): + # Early skip directories if they match any of the patterns + if self._should_skip_dir(rel_path): self.print_debug(f'Skipping directory: {rel_path}') dirnames.clear() continue @@ -268,7 +267,7 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: if file_size < self.min_size or file_size > self.max_size: self.print_debug(f'Skipping file: {file_rel_path} (size: {file_size})') continue - if path_spec.match_file(file_rel_path): + if self.path_spec.match_file(file_rel_path): self.print_debug(f'Skipping file: {file_rel_path}') continue else: @@ -276,9 +275,7 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: return files - def _create_skip_path_matchers(self) -> Tuple[PathSpec, Set[str]]: - dir_patterns = {p.rstrip('/') for p in self.skip_patterns if p.endswith('/')} - - path_spec = PathSpec.from_lines('gitwildmatch', self.skip_patterns) - - return path_spec, dir_patterns + def _should_skip_dir(self, dir_rel_path: str) -> bool: + return any(dir_rel_path.startswith(p) for p in self.skip_patterns) or self.path_spec.match_file( + dir_rel_path + '/' + ) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 2ac1f955..aa17058c 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -401,7 +401,6 @@ def scan_folder(self, scan_dir: str) -> bool: if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') - scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir) + 1 self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None if not self.quiet and self.isatty: From f351b13978458ee49c36cce805e357ca6cc8515e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 20 Nov 2024 14:11:20 +0100 Subject: [PATCH 225/489] feat: SP-1856 Update changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b4000f1..cd0a74ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use scanoss.json as default settings file if no argument is supplied - Add —skip-settings-file flag - Update scanoss settings schema to allow skipping specific folders, files, and extensions +- Add ScanFilter class to handle filtering of files and folders based on settings ## [1.18.1] - 2024-11-19 ### Added @@ -416,5 +417,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.17.4]: https://github.com/scanoss/scanoss.py/compare/v1.17.3...v1.17.4 [1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 [1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 -[1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.1 -[1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.0...v1.19.0 \ No newline at end of file +[1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.18.0...v1.18.1 +[1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 \ No newline at end of file From 057519ed0d58cc581f43023ef4bbed953ce6d8b0 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 20 Nov 2024 14:48:19 +0100 Subject: [PATCH 226/489] feat: SP-1856 Fix and add tests --- requirements.txt | 3 +- setup.cfg | 1 + tests/test_scan_filter.py | 102 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 tests/test_scan_filter.py diff --git a/requirements.txt b/requirements.txt index 0b95ea31..76350722 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ urllib3 pyOpenSSL google-api-core importlib_resources -packageurl-python \ No newline at end of file +packageurl-python +pathspec \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 8de3fd05..aa0c4f03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,6 +36,7 @@ install_requires = google-api-core importlib_resources packageurl-python + pathspec [options.extras_require] diff --git a/tests/test_scan_filter.py b/tests/test_scan_filter.py new file mode 100644 index 00000000..65c2bd50 --- /dev/null +++ b/tests/test_scan_filter.py @@ -0,0 +1,102 @@ +import unittest +from unittest.mock import patch + +from scanoss.scan_filter import ScanFilter + + +class TestScanFilter(unittest.TestCase): + def setUp(self): + self.settings = { + "skip": {"patterns": [], "sizes": {"min": 0, "max": float("inf")}} + } + self.scan_filter = ScanFilter(settings=self.settings, debug=True) + + @patch("os.walk") + @patch("os.path.getsize") + def test_default_extensions(self, mock_getsize, mock_walk): + mock_walk.return_value = [ + ("/scan_root", ["dir1", "dir2"], ["file1.go", "file2.js"]), + ("/scan_root/dir1", [], ["file3.py", "file4.go"]), + ("/scan_root/dir2", [], ["file5.js", "file6.png"]), + ] + mock_getsize.side_effect = [100, 200, 300, 400, 500, 600] + + # All the other files should be removed by the filter because they are in default skipped extensions + expected_files = [ + "./file1.go", + "./file2.js", + "dir1/file3.py", + "dir1/file4.go", + "dir2/file5.js", + ] + + filtered_files = self.scan_filter.get_filtered_files("/scan_root") + self.assertEqual(filtered_files, expected_files) + + @patch("os.walk") + @patch("os.path.getsize") + def test_default_folders(self, mock_getsize, mock_walk): + mock_walk.return_value = [ + ("/scan_root", ["__pycache__", "dir1"], []), + ("/scan_root/__pycache__", [], ["file1.pyc", "file2.pyc"]), + ("/scan_root/dir1", ["nbdist"], ["file3.py", "file4.go"]), + ("/scan_root/dir1/nbdist", [], ["test.py", "test1.py"]), + ] + mock_getsize.side_effect = [100, 200, 300, 400, 500, 600] + + # All the other files should be removed by the filter because they are in default skipped extensions + expected_files = [ + "dir1/file3.py", + "dir1/file4.go", + ] + + filtered_files = self.scan_filter.get_filtered_files("/scan_root") + self.assertEqual(filtered_files, expected_files) + + @patch("os.walk") + @patch("os.path.getsize") + def test_skip_files_by_size(self, mock_getsize, mock_walk): + self.scan_filter.min_size = 150 + self.scan_filter.max_size = 450 + + mock_walk.return_value = [ + ("/scan_root", [], ["file1.js", "file2.go", "file3.py"]), + ] + mock_getsize.side_effect = [100, 200, 300] + + expected_files = ["./file2.go", "./file3.py"] + + filtered_files = self.scan_filter.get_filtered_files("/scan_root") + self.assertEqual(filtered_files, expected_files) + + @patch("os.walk") + def test_skip_directories(self, mock_walk): + mock_walk.return_value = [ + ("/scan_root", ["dir1", "dir2"], ["file1.js"]), + ("/scan_root/dir1", [], ["file2.js"]), + ("/scan_root/dir2", [], ["file3.py"]), + ] + + self.scan_filter.skip_patterns.append("dir2/") + + expected_files = ["./file1.js", "dir1/file2.js"] + + filtered_files = self.scan_filter.get_filtered_files("/scan_root") + self.assertEqual(filtered_files, expected_files) + + @patch("os.walk") + def test_custom_skip_patterns(self, mock_walk): + self.scan_filter.skip_patterns.append("*.md") + + mock_walk.return_value = [ + ("/scan_root", [], ["file1.txt", "file2.md", "file3.py"]), + ] + + expected_files = ["./file1.txt", "./file3.py"] + + filtered_files = self.scan_filter.get_filtered_files("/scan_root") + self.assertEqual(filtered_files, expected_files) + + +if __name__ == "__main__": + unittest.main() From b156a246ce20bd87d41a608b72acf2b41ff8ac93 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 20 Nov 2024 15:08:36 +0100 Subject: [PATCH 227/489] feat: SP-1856 Handle when scanoss settings file is skipped or does not exist --- src/scanoss/scan_filter.py | 24 ++++++++++++++++-------- src/scanoss/scanner.py | 13 ++++++++++++- src/scanoss/scanoss_settings.py | 26 ++++++++++++++++---------- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index b7a7c4e9..78146786 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -3,6 +3,7 @@ from pathspec import PathSpec +from scanoss.scanoss_settings import ScanossSettings from scanoss.scanossbase import ScanossBase DEFAULT_SKIPPED_FILES = { @@ -210,29 +211,36 @@ def __init__( debug: bool = False, trace: bool = False, quiet: bool = False, - settings: dict = None, + scanoss_settings: 'ScanossSettings' = None, ): """ - Initialize filter with settings from ScanossSettings. + Initialize scan filters based on default settings. Optionally append custom settings. Args: - settings (ScanossSettings): Settings instance containing scan configuration + debug (bool): Enable debug output + trace (bool): Enable trace output + quiet (bool): Suppress output + scanoss_settings (ScanossSettings): Custom settings to override defaults """ super().__init__(debug, trace, quiet) - skip = settings.get('skip', {}) + self.min_size = 0 + self.max_size = float('inf') + skip_patterns = [] skip_patterns.extend(DEFAULT_SKIPPED_FILES) - skip_patterns.extend(f'{dir}/' for dir in DEFAULT_SKIPPED_DIRS) + skip_patterns.extend(f'{dir_path}/' for dir_path in DEFAULT_SKIPPED_DIRS) skip_patterns.extend(f'*{ext}' for ext in DEFAULT_SKIPPED_EXT) skip_patterns.extend(f'*{ext}/' for ext in DEFAULT_SKIPPED_DIR_EXT) - skip_patterns.extend(skip.get('patterns', [])) + + if scanoss_settings: + skip_patterns.extend(scanoss_settings.get_skip_patterns()) + self.min_size = scanoss_settings.get_skip_sizes().get('min', 0) + self.max_size = scanoss_settings.get_skip_sizes().get('max', float('inf')) self.skip_patterns = skip_patterns self.path_spec = PathSpec.from_lines('gitwildmatch', self.skip_patterns) - self.min_size = skip.get('sizes', {}).get('min', 0) - self.max_size = skip.get('sizes', {}).get('max', float('inf')) def get_filtered_files(self, root: str) -> List[str]: """Get a list of files to scan based on the filter settings. diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index aa17058c..976db080 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -33,6 +33,8 @@ from progress.spinner import Spinner from pypac.parser import PACFile +from scanoss.scan_filter import ScanFilter + from .scanossapi import ScanossApi from .cyclonedx import CycloneDx from .spdxlite import SpdxLite @@ -191,6 +193,13 @@ def __init__( self.scan_settings = scan_settings self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None self._maybe_set_api_sbom() + + self.scan_filters = ScanFilter( + debug=self.debug, + trace=self.trace, + quiet=self.quiet, + scanoss_settings=self.scan_settings, + ) def _maybe_set_api_sbom(self): if not self.scan_settings: @@ -414,7 +423,9 @@ def scan_folder(self, scan_dir: str) -> bool: wfp_file_count = 0 # count number of files in each queue post scan_started = False - to_scan_files = self.scan_settings.get_filtered_files(scan_dir) + + to_scan_files = self.scan_filters.get_filtered_files(scan_dir) + for to_scan_file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index cb463e82..7cd226bc 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -26,7 +26,6 @@ from pathlib import Path from typing import List, TypedDict -from scanoss.scan_filter import ScanFilter from scanoss.utils.file import validate_json_file from .scanossbase import ScanossBase @@ -47,6 +46,7 @@ class ScanossSettings(ScanossBase): """ Handles the loading and parsing of the SCANOSS settings file """ + def __init__( self, debug: bool = False, @@ -66,7 +66,6 @@ def __init__( self.data = {} self.settings_file_type = None self.scan_type = None - self.filter = None if filepath: self.load_json_file(filepath) @@ -91,12 +90,6 @@ def load_json_file(self, filepath: str) -> 'ScanossSettings': if not result.is_valid: raise ScanossSettingsError(f'Problem with settings file. {result.error}') self.data = result.data - self.filter = ScanFilter( - debug=self.debug, - quiet=self.quiet, - trace=self.trace, - settings=self.data.get('settings', {}), - ) self.print_debug(f'Loading scan settings from: {filepath}') return self @@ -249,5 +242,18 @@ def is_legacy(self): """Check if the settings file is legacy""" return self.settings_file_type == 'legacy' - def get_filtered_files(self, scan_root: str) -> List[str]: - return self.filter.get_filtered_files(scan_root) + def get_skip_patterns(self) -> List[str]: + """ + Get the list of patterns to skip + Returns: + List: List of patterns to skip + """ + return self.data.get('settings', {}).get('skip', {}).get('patterns', []) + + def get_skip_sizes(self) -> dict: + """ + Get the min and max sizes to skip + Returns: + dict: Min and max sizes to skip + """ + return self.data.get('settings', {}).get('skip', {}).get('sizes', {}) From 8c7758097c42f534a0d36b6cf981af00109f72e8 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 20 Nov 2024 16:01:06 +0100 Subject: [PATCH 228/489] feat: SP-1856 Fix tests --- src/scanoss/scan_filter.py | 2 +- tests/test_scan_filter.py | 93 ++++++++++++++++++++------------------ 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index 78146786..750f69e4 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -284,6 +284,6 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: return files def _should_skip_dir(self, dir_rel_path: str) -> bool: - return any(dir_rel_path.startswith(p) for p in self.skip_patterns) or self.path_spec.match_file( + return any(dir_rel_path == p.rstrip('/') for p in self.skip_patterns) or self.path_spec.match_file( dir_rel_path + '/' ) diff --git a/tests/test_scan_filter.py b/tests/test_scan_filter.py index 65c2bd50..cacf3657 100644 --- a/tests/test_scan_filter.py +++ b/tests/test_scan_filter.py @@ -6,97 +6,100 @@ class TestScanFilter(unittest.TestCase): def setUp(self): - self.settings = { - "skip": {"patterns": [], "sizes": {"min": 0, "max": float("inf")}} - } - self.scan_filter = ScanFilter(settings=self.settings, debug=True) + self.scan_filter = ScanFilter(debug=True) - @patch("os.walk") - @patch("os.path.getsize") + @patch('os.walk') + @patch('os.path.getsize') def test_default_extensions(self, mock_getsize, mock_walk): mock_walk.return_value = [ - ("/scan_root", ["dir1", "dir2"], ["file1.go", "file2.js"]), - ("/scan_root/dir1", [], ["file3.py", "file4.go"]), - ("/scan_root/dir2", [], ["file5.js", "file6.png"]), + ('/scan_root', ['dir1', 'dir2'], ['file1.go', 'file2.js']), + ('/scan_root/dir1', [], ['file3.py', 'file4.go']), + ('/scan_root/dir2', [], ['file5.js', 'file6.png']), ] mock_getsize.side_effect = [100, 200, 300, 400, 500, 600] # All the other files should be removed by the filter because they are in default skipped extensions expected_files = [ - "./file1.go", - "./file2.js", - "dir1/file3.py", - "dir1/file4.go", - "dir2/file5.js", + './file1.go', + './file2.js', + 'dir1/file3.py', + 'dir1/file4.go', + 'dir2/file5.js', ] - filtered_files = self.scan_filter.get_filtered_files("/scan_root") + filtered_files = self.scan_filter.get_filtered_files('/scan_root') self.assertEqual(filtered_files, expected_files) - @patch("os.walk") - @patch("os.path.getsize") + @patch('os.walk') + @patch('os.path.getsize') def test_default_folders(self, mock_getsize, mock_walk): mock_walk.return_value = [ - ("/scan_root", ["__pycache__", "dir1"], []), - ("/scan_root/__pycache__", [], ["file1.pyc", "file2.pyc"]), - ("/scan_root/dir1", ["nbdist"], ["file3.py", "file4.go"]), - ("/scan_root/dir1/nbdist", [], ["test.py", "test1.py"]), + ('/scan_root', ['__pycache__', 'dir1'], []), + ('/scan_root/__pycache__', [], ['file1.pyc', 'file2.pyc']), + ('/scan_root/dir1', ['nbdist'], ['file3.py', 'file4.go']), + ('/scan_root/dir1/nbdist', [], ['test.py', 'test1.py']), ] mock_getsize.side_effect = [100, 200, 300, 400, 500, 600] # All the other files should be removed by the filter because they are in default skipped extensions expected_files = [ - "dir1/file3.py", - "dir1/file4.go", + 'dir1/file3.py', + 'dir1/file4.go', ] - filtered_files = self.scan_filter.get_filtered_files("/scan_root") + filtered_files = self.scan_filter.get_filtered_files('/scan_root') self.assertEqual(filtered_files, expected_files) - @patch("os.walk") - @patch("os.path.getsize") + @patch('os.walk') + @patch('os.path.getsize') def test_skip_files_by_size(self, mock_getsize, mock_walk): self.scan_filter.min_size = 150 self.scan_filter.max_size = 450 mock_walk.return_value = [ - ("/scan_root", [], ["file1.js", "file2.go", "file3.py"]), + ('/scan_root', [], ['file1.js', 'file2.go', 'file3.py']), ] mock_getsize.side_effect = [100, 200, 300] - expected_files = ["./file2.go", "./file3.py"] + expected_files = ['./file2.go', './file3.py'] - filtered_files = self.scan_filter.get_filtered_files("/scan_root") + filtered_files = self.scan_filter.get_filtered_files('/scan_root') self.assertEqual(filtered_files, expected_files) - @patch("os.walk") - def test_skip_directories(self, mock_walk): + @patch('os.walk') + @patch('os.path.getsize') + def test_skip_directories(self, mock_getsize, mock_walk): mock_walk.return_value = [ - ("/scan_root", ["dir1", "dir2"], ["file1.js"]), - ("/scan_root/dir1", [], ["file2.js"]), - ("/scan_root/dir2", [], ["file3.py"]), + ('/scan_root', ['dir1', 'dir2'], ['file1.js']), + ('/scan_root/dir1', [], ['file2.js']), + ('/scan_root/dir2', [], ['file3.py']), ] - self.scan_filter.skip_patterns.append("dir2/") + mock_getsize.side_effect = [100, 200, 300] - expected_files = ["./file1.js", "dir1/file2.js"] + self.scan_filter.skip_patterns.append('dir2/') - filtered_files = self.scan_filter.get_filtered_files("/scan_root") - self.assertEqual(filtered_files, expected_files) + expected_files = ['./file1.js', 'dir1/file2.js'] - @patch("os.walk") - def test_custom_skip_patterns(self, mock_walk): - self.scan_filter.skip_patterns.append("*.md") + filtered_files = self.scan_filter.get_filtered_files('/scan_root') + self.assertEqual(filtered_files, expected_files) + @patch('os.walk') + @patch('os.path.getsize') + def test_custom_skip_patterns(self, mock_getsize, mock_walk): mock_walk.return_value = [ - ("/scan_root", [], ["file1.txt", "file2.md", "file3.py"]), + ('/scan_root', [], ['file1.txt', 'file2.md', 'file3.py', 'file4.rst']), ] - expected_files = ["./file1.txt", "./file3.py"] + mock_getsize.side_effect = [100, 200, 300, 400] + + self.scan_filter.skip_patterns.append('*.rst') + + expected_files = ['./file3.py'] - filtered_files = self.scan_filter.get_filtered_files("/scan_root") + filtered_files = self.scan_filter.get_filtered_files('/scan_root') self.assertEqual(filtered_files, expected_files) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() From 9b5dd9351db9aae83f7c3341a873c3ea10f77149 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 20 Nov 2024 16:28:14 +0100 Subject: [PATCH 229/489] feat: SP-1856 Add support for skipping default filtered extensions, files, hidden folders, etc --- src/scanoss/scan_filter.py | 22 +++++++++++++++++----- src/scanoss/scanner.py | 3 +++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index 750f69e4..ddaf0950 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -212,6 +212,9 @@ def __init__( trace: bool = False, quiet: bool = False, scanoss_settings: 'ScanossSettings' = None, + all_extensions: bool = False, + all_folders: bool = False, + hidden_files_folders: bool = False, ): """ Initialize scan filters based on default settings. Optionally append custom settings. @@ -221,18 +224,24 @@ def __init__( trace (bool): Enable trace output quiet (bool): Suppress output scanoss_settings (ScanossSettings): Custom settings to override defaults + all_extensions (bool): Include all file extensions + all_folders (bool): Include all folders + hidden_files_folders (bool): Include hidden files and folders """ super().__init__(debug, trace, quiet) self.min_size = 0 self.max_size = float('inf') + self.hidden_files_folders = hidden_files_folders skip_patterns = [] skip_patterns.extend(DEFAULT_SKIPPED_FILES) - skip_patterns.extend(f'{dir_path}/' for dir_path in DEFAULT_SKIPPED_DIRS) - skip_patterns.extend(f'*{ext}' for ext in DEFAULT_SKIPPED_EXT) - skip_patterns.extend(f'*{ext}/' for ext in DEFAULT_SKIPPED_DIR_EXT) + if not all_extensions: + skip_patterns.extend(f'*{ext}' for ext in DEFAULT_SKIPPED_EXT) + skip_patterns.extend(f'*{ext}/' for ext in DEFAULT_SKIPPED_DIR_EXT) + if not all_folders: + skip_patterns.extend(f'{dir_path}/' for dir_path in DEFAULT_SKIPPED_DIRS) if scanoss_settings: skip_patterns.extend(scanoss_settings.get_skip_patterns()) @@ -284,6 +293,9 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: return files def _should_skip_dir(self, dir_rel_path: str) -> bool: - return any(dir_rel_path == p.rstrip('/') for p in self.skip_patterns) or self.path_spec.match_file( - dir_rel_path + '/' + is_hidden = dir_rel_path != '.' and any(part.startswith('.') for part in dir_rel_path.split(os.sep)) + return ( + (is_hidden and not self.hidden_files_folders) + or any(dir_rel_path == p.rstrip('/') for p in self.skip_patterns) + or self.path_spec.match_file(dir_rel_path + '/') ) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 976db080..b78f903c 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -199,6 +199,9 @@ def __init__( trace=self.trace, quiet=self.quiet, scanoss_settings=self.scan_settings, + all_extensions=all_extensions, + all_folders=all_folders, + hidden_files_folders=hidden_files_folders, ) def _maybe_set_api_sbom(self): From df5d105bfa349e0ebb542f487bd1712fa6f8b30c Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 20 Nov 2024 16:37:38 +0100 Subject: [PATCH 230/489] feat: SP-1856 Update docs --- docs/source/scanoss_settings_schema.rst | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index 85521a3f..23350c37 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -273,9 +273,28 @@ Here's a complete example showing all sections: { "self": { "name": "example-project", - "license": "Apache-2.0", + "license": "Apache-2.0", "description": "Example project configuration" }, + "settings": { + "skip": { + "patterns": [ + "node_modules/", + "dist/", + "build/", + "*.log", + "!important.log", + "temp/", + "*.tmp", + "debug[0-9]*.txt", + "**/*test.js" + ], + "sizes": { + "min": 512, + "max": 5242880 + } + } + }, "bom": { "include": [ { @@ -287,7 +306,7 @@ Here's a complete example showing all sections: "remove": [ { "purl": "pkg:npm/deprecated-pkg@1.0.0", - "comment": "Remove deprecated package" + "comment": "Remove deprecated package" } ], "replace": [ From 2d8bb460ac403314dbdd7c4b688f0fa02a44c79a Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 2 Dec 2024 12:19:02 +0100 Subject: [PATCH 231/489] feat: SP-1856 Add json schema validation --- .gitignore | 1 + .../_static/scanoss-settings-schema.json | 3 +- requirements.txt | 3 +- scanoss-settings-schema.json | 177 ++++++++++++++++++ setup.cfg | 1 + src/scanoss/scanoss_settings.py | 9 + 6 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 scanoss-settings-schema.json diff --git a/.gitignore b/.gitignore index 8e474346..6d384760 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ docs/build !tests/data/*.json !docs/source/_static/*.json +!scanoss-settings-schema.json diff --git a/docs/source/_static/scanoss-settings-schema.json b/docs/source/_static/scanoss-settings-schema.json index 575af6bb..2a75ffa4 100644 --- a/docs/source/_static/scanoss-settings-schema.json +++ b/docs/source/_static/scanoss-settings-schema.json @@ -38,10 +38,8 @@ "path/to/folder", "path/to/folder/**", "path/to/folder/**/*", - "path/to/file.c", "path/to/another/file.py", - "**/*.ts", "**/*.json" ] @@ -176,3 +174,4 @@ } } } + diff --git a/requirements.txt b/requirements.txt index 76350722..0b464ec4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ pyOpenSSL google-api-core importlib_resources packageurl-python -pathspec \ No newline at end of file +pathspec +jsonschema \ No newline at end of file diff --git a/scanoss-settings-schema.json b/scanoss-settings-schema.json new file mode 100644 index 00000000..2a75ffa4 --- /dev/null +++ b/scanoss-settings-schema.json @@ -0,0 +1,177 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "Scanoss Settings", + "type": "object", + "properties": { + "self": { + "type": "object", + "description": "Description of the project under analysis", + "properties": { + "name": { + "type": "string", + "description": "Name of the project" + }, + "license": { + "type": "string", + "description": "License of the project" + }, + "description": { + "type": "string", + "description": "Description of the project" + } + } + }, + "settings": { + "type": "object", + "description": "Scan settings and other configurations", + "properties": { + "skip": { + "type": "object", + "description": "Set of rules to skip files from the scan", + "properties": { + "patterns": { + "type": "array", + "description": "List of glob patterns to skip files", + "items": { + "type": "string", + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + }, + "uniqueItems": true + }, + "sizes": { + "type": "object", + "description": "Set of rules to skip files based on their size", + "properties": { + "min": { + "type": "integer", + "description": "Minimum size of the file in bytes" + }, + "max": { + "type": "integer", + "description": "Maximum size of the file in bytes" + } + }, + "required": ["max"] + } + } + } + } + }, + "bom": { + "type": "object", + "description": "BOM Rules: Set of rules that will be used to modify the BOM before and after the scan is completed", + "properties": { + "include": { + "type": "array", + "description": "Set of rules to be added as context when scanning. This list will be sent as payload to the API.", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path", + "examples": ["/path/to/file", "/path/to/another/file"], + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "purl": { + "type": "string", + "description": "Package URL to be used to match the component", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + }, + "comment": { + "type": "string", + "description": "Additional notes or comments" + } + }, + "uniqueItems": true, + "required": ["purl"] + } + }, + "remove": { + "type": "array", + "description": "Set of rules that will remove files from the results file after the scan is completed.", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path", + "examples": ["/path/to/file", "/path/to/another/file"] + }, + "purl": { + "type": "string", + "description": "Package URL", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + }, + "comment": { + "type": "string", + "description": "Additional notes or comments" + } + }, + "uniqueItems": true, + "required": ["purl"] + } + }, + "replace": { + "type": "array", + "description": "Set of rules that will replace components with the specified one after the scan is completed.", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path", + "examples": ["/path/to/file", "/path/to/another/file"] + }, + "purl": { + "type": "string", + "description": "Package URL to replace", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + }, + "comment": { + "type": "string", + "description": "Additional notes or comments" + }, + "license": { + "type": "string", + "description": "License of the component. Should be a valid SPDX license expression", + "examples": ["MIT", "Apache-2.0"] + }, + "replace_with": { + "type": "string", + "description": "Package URL to replace with", + "examples": [ + "pkg:npm/vue@2.6.12", + "pkg:golang/github.com/golang/go@1.17.3" + ] + } + }, + "uniqueItems": true, + "required": ["purl", "replace_with"] + } + } + } + } + } +} + diff --git a/setup.cfg b/setup.cfg index aa0c4f03..086a1fde 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ install_requires = importlib_resources packageurl-python pathspec + jsonschema [options.extras_require] diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 7cd226bc..0eb06277 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -26,6 +26,8 @@ from pathlib import Path from typing import List, TypedDict +from jsonschema import validate + from scanoss.utils.file import validate_json_file from .scanossbase import ScanossBase @@ -67,6 +69,9 @@ def __init__( self.settings_file_type = None self.scan_type = None + with open('scanoss-settings-schema.json') as f: + self.schema = json.load(f) + if filepath: self.load_json_file(filepath) @@ -89,6 +94,10 @@ def load_json_file(self, filepath: str) -> 'ScanossSettings': result = validate_json_file(json_file) if not result.is_valid: raise ScanossSettingsError(f'Problem with settings file. {result.error}') + try: + validate(result.data, self.schema) + except Exception as e: + raise ScanossSettingsError(f'Invalid settings file. {e}') from e self.data = result.data self.print_debug(f'Loading scan settings from: {filepath}') return self From 6db200155e9a67392ec28ac0c8653341d751f90d Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Dec 2024 06:57:13 +0100 Subject: [PATCH 232/489] feat: SP-1856 Use scan filters in wfp_folder --- src/scanoss/scanner.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index b78f903c..94c66c42 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -1074,32 +1074,22 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): Fingerprint the specified folder producing fingerprints """ if not scan_dir: - raise Exception(f"ERROR: Please specify a folder to fingerprint") + raise Exception(f'ERROR: Please specify a folder to fingerprint') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): - raise Exception(f"ERROR: Specified folder does not exist or is not a folder: {scan_dir}") + raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') wfps = '' - scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir) + 1 + self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') - for root, dirs, files in os.walk(scan_dir): - dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories - filtered_files = self.__filter_files(files) # Strip out unwanted files - self.print_trace(f'Root: {root}, Dirs: {dirs}, Files {filtered_files}') - for file in filtered_files: - path = os.path.join(root, file) - f_size = 0 - try: - f_size = os.stat(path).st_size - except Exception as e: - self.print_trace( - f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink - if f_size > 0: # Ignore empty files - self.print_debug(f'Fingerprinting {path}...') - if spinner: - spinner.next() - wfps += self.winnowing.wfp_for_file(path, Scanner.__strip_dir(scan_dir, scan_dir_len, path)) + + to_fingerprint_files = self.scan_filters.get_filtered_files(scan_dir) + for file in to_fingerprint_files: + if spinner: + spinner.next() + abs_path = Path(scan_dir, file).resolve() + wfps += self.winnowing.wfp_for_file(str(abs_path), file) if spinner: spinner.finish() if wfps: From e82f29d3baa00941c1beff85c408be150295bd5a Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Dec 2024 07:28:04 +0100 Subject: [PATCH 233/489] feat: SP-1856 Use scan filters in scan_files --- src/scanoss/scan_filter.py | 57 ++++++++++++++++++++++++++++++-------- src/scanoss/scanner.py | 27 +++++------------- tests/test_scan_filter.py | 10 +++---- 3 files changed, 57 insertions(+), 37 deletions(-) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index ddaf0950..46565374 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from typing import List from pathspec import PathSpec @@ -251,8 +252,8 @@ def __init__( self.skip_patterns = skip_patterns self.path_spec = PathSpec.from_lines('gitwildmatch', self.skip_patterns) - def get_filtered_files(self, root: str) -> List[str]: - """Get a list of files to scan based on the filter settings. + def get_filtered_files_from_folder(self, root: str) -> List[str]: + """Retrieve a list of files to scan or fingerprint from a given directory root based on filter settings. Args: root (str): Root directory to scan @@ -263,37 +264,69 @@ def get_filtered_files(self, root: str) -> List[str]: files = self._walk_with_ignore(root) return files + def get_filtered_files_from_files(self, files: List[str]) -> List[str]: + """Retrieve a list of files to scan or fingerprint from a given list of files based on filter settings. + + Args: + files (List[str]): List of files to scan + + Returns: + list[str]: List of files to scan + """ + filtered_files = [] + for file in files: + file_path = Path(file).resolve() + file_rel_path = file_path.relative_to(Path.cwd()) + + if not file_path.exists(): + self.print_debug(f'Skipping file: {file_rel_path} (does not exist)') + continue + + file_size = file_path.stat().st_size + + if file_size < self.min_size or file_size > self.max_size: + self.print_debug(f'Skipping file: {file} (size: {file_size})') + continue + + if self.path_spec.match_file(str(file_rel_path)): + self.print_debug(f'Skipping file: {file}') + continue + + filtered_files.append(str(file)) + return filtered_files + def _walk_with_ignore(self, scan_root: str) -> List[str]: files = [] - root = os.path.abspath(scan_root) + root = Path(scan_root).resolve() for dirpath, dirnames, filenames in os.walk(root): - rel_path = os.path.relpath(dirpath, root) + dirpath = Path(dirpath) + rel_path = dirpath.relative_to(root) - # Early skip directories if they match any of the patterns - if self._should_skip_dir(rel_path): + if self._should_skip_dir(str(rel_path)): self.print_debug(f'Skipping directory: {rel_path}') dirnames.clear() continue for filename in filenames: - file_rel_path = os.path.join(rel_path, filename) - file_path = os.path.join(dirpath, filename) - file_size = os.path.getsize(file_path) + file_path = dirpath / filename + file_rel_path = rel_path / filename + file_size = file_path.stat().st_size if file_size < self.min_size or file_size > self.max_size: self.print_debug(f'Skipping file: {file_rel_path} (size: {file_size})') continue - if self.path_spec.match_file(file_rel_path): + if self.path_spec.match_file(str(file_rel_path)): self.print_debug(f'Skipping file: {file_rel_path}') continue else: - files.append(file_rel_path) + files.append(str(file_rel_path)) return files def _should_skip_dir(self, dir_rel_path: str) -> bool: - is_hidden = dir_rel_path != '.' and any(part.startswith('.') for part in dir_rel_path.split(os.sep)) + dir_path = Path(dir_rel_path) + is_hidden = dir_path != Path('.') and any(part.startswith('.') for part in dir_path.parts) return ( (is_hidden and not self.hidden_files_folders) or any(dir_rel_path == p.rstrip('/') for p in self.skip_patterns) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 94c66c42..d8296c1c 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -427,7 +427,7 @@ def scan_folder(self, scan_dir: str) -> bool: scan_started = False - to_scan_files = self.scan_filters.get_filtered_files(scan_dir) + to_scan_files = self.scan_filters.get_filtered_files_from_folder(scan_dir) for to_scan_file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): @@ -670,23 +670,10 @@ def scan_files(self, files: []) -> bool: file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False - filtered_files = [] - # Filter the files to remove anything we shouldn't scan - for file in files: - filename = os.path.basename(file) - filtered_filenames = self.__filter_files([filename]) - if not filtered_filenames or len(filtered_filenames) == 0: - self.print_debug(f'Skipping filtered file: {file}') - continue - paths = os.path.dirname(file).split(os.sep) - if len(self.__filter_dirs(paths)) == len(paths): # Nothing found to filter - filtered_files.append(file) - else: - self.print_debug(f'Skipping filtered (folder) file: {file}') - if len(filtered_files) > 0: - self.print_debug(f'Scanning {len(filtered_files)} files...') - # Process all the requested files - for file in filtered_files: + + to_scan_files = self.scan_filters.get_filtered_files_from_files(files) + + for file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break @@ -746,7 +733,7 @@ def scan_files(self, files: []) -> bool: if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: - Scanner.print_stderr(f'Warning: No files found to scan from: {filtered_files}') + Scanner.print_stderr(f'Warning: No files found to scan from: {to_scan_files}') return success def scan_files_with_options(self, files: [], deps_file: str = None, file_map: dict = None) -> bool: @@ -1084,7 +1071,7 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') - to_fingerprint_files = self.scan_filters.get_filtered_files(scan_dir) + to_fingerprint_files = self.scan_filters.get_filtered_files_from_folder(scan_dir) for file in to_fingerprint_files: if spinner: spinner.next() diff --git a/tests/test_scan_filter.py b/tests/test_scan_filter.py index cacf3657..54410c99 100644 --- a/tests/test_scan_filter.py +++ b/tests/test_scan_filter.py @@ -27,7 +27,7 @@ def test_default_extensions(self, mock_getsize, mock_walk): 'dir2/file5.js', ] - filtered_files = self.scan_filter.get_filtered_files('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') self.assertEqual(filtered_files, expected_files) @patch('os.walk') @@ -47,7 +47,7 @@ def test_default_folders(self, mock_getsize, mock_walk): 'dir1/file4.go', ] - filtered_files = self.scan_filter.get_filtered_files('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') self.assertEqual(filtered_files, expected_files) @patch('os.walk') @@ -63,7 +63,7 @@ def test_skip_files_by_size(self, mock_getsize, mock_walk): expected_files = ['./file2.go', './file3.py'] - filtered_files = self.scan_filter.get_filtered_files('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') self.assertEqual(filtered_files, expected_files) @patch('os.walk') @@ -81,7 +81,7 @@ def test_skip_directories(self, mock_getsize, mock_walk): expected_files = ['./file1.js', 'dir1/file2.js'] - filtered_files = self.scan_filter.get_filtered_files('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') self.assertEqual(filtered_files, expected_files) @patch('os.walk') @@ -97,7 +97,7 @@ def test_custom_skip_patterns(self, mock_getsize, mock_walk): expected_files = ['./file3.py'] - filtered_files = self.scan_filter.get_filtered_files('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') self.assertEqual(filtered_files, expected_files) From 0a302efc5fc8e686817df57ca5c28d4f73c38359 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Dec 2024 07:46:16 +0100 Subject: [PATCH 234/489] feat: SP-1856 Fix unit tests --- CHANGELOG.md | 2 +- tests/test_scan_filter.py | 122 +++++++++++++++++++++----------------- 2 files changed, 69 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd0a74ca..a8fdb7e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -418,4 +418,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 [1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 [1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.18.0...v1.18.1 -[1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 \ No newline at end of file +[1.19.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 \ No newline at end of file diff --git a/tests/test_scan_filter.py b/tests/test_scan_filter.py index 54410c99..d7cb17b2 100644 --- a/tests/test_scan_filter.py +++ b/tests/test_scan_filter.py @@ -1,5 +1,7 @@ +import os +import shutil +import tempfile import unittest -from unittest.mock import patch from scanoss.scan_filter import ScanFilter @@ -7,97 +9,109 @@ class TestScanFilter(unittest.TestCase): def setUp(self): self.scan_filter = ScanFilter(debug=True) - - @patch('os.walk') - @patch('os.path.getsize') - def test_default_extensions(self, mock_getsize, mock_walk): - mock_walk.return_value = [ - ('/scan_root', ['dir1', 'dir2'], ['file1.go', 'file2.js']), - ('/scan_root/dir1', [], ['file3.py', 'file4.go']), - ('/scan_root/dir2', [], ['file5.js', 'file6.png']), + self.test_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_dir) + + def create_files(self, files): + for file in files: + file_path = os.path.join(self.test_dir, file) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'w') as f: + f.write('test') + + def test_default_extensions(self): + files = [ + 'file1.go', + 'file2.js', + 'dir1/file3.py', + 'dir1/file4.go', + 'dir2/file5.js', + 'dir2/file6.png', ] - mock_getsize.side_effect = [100, 200, 300, 400, 500, 600] + self.create_files(files) - # All the other files should be removed by the filter because they are in default skipped extensions expected_files = [ - './file1.go', - './file2.js', + 'file2.js', + 'file1.go', + 'dir2/file5.js', 'dir1/file3.py', 'dir1/file4.go', - 'dir2/file5.js', ] - filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) self.assertEqual(filtered_files, expected_files) - @patch('os.walk') - @patch('os.path.getsize') - def test_default_folders(self, mock_getsize, mock_walk): - mock_walk.return_value = [ - ('/scan_root', ['__pycache__', 'dir1'], []), - ('/scan_root/__pycache__', [], ['file1.pyc', 'file2.pyc']), - ('/scan_root/dir1', ['nbdist'], ['file3.py', 'file4.go']), - ('/scan_root/dir1/nbdist', [], ['test.py', 'test1.py']), + def test_default_folders(self): + files = [ + '__pycache__/file1.pyc', + '__pycache__/file2.pyc', + 'dir1/nbdist/test.py', + 'dir1/nbdist/test1.py', + 'dir1/file3.py', + 'dir1/file4.go', ] - mock_getsize.side_effect = [100, 200, 300, 400, 500, 600] + self.create_files(files) - # All the other files should be removed by the filter because they are in default skipped extensions expected_files = [ 'dir1/file3.py', 'dir1/file4.go', ] - filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) self.assertEqual(filtered_files, expected_files) - @patch('os.walk') - @patch('os.path.getsize') - def test_skip_files_by_size(self, mock_getsize, mock_walk): + def test_skip_files_by_size(self): self.scan_filter.min_size = 150 self.scan_filter.max_size = 450 - mock_walk.return_value = [ - ('/scan_root', [], ['file1.js', 'file2.go', 'file3.py']), + files = [ + 'file1.js', + 'file2.go', + 'file3.py', ] - mock_getsize.side_effect = [100, 200, 300] + self.create_files(files) + + for file in files: + file_path = os.path.join(self.test_dir, file) + with open(file_path, 'w') as f: + f.write('a' * (100 if 'file1' in file else 200 if 'file2' in file else 300)) - expected_files = ['./file2.go', './file3.py'] + expected_files = ['file3.py', 'file2.go'] - filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) self.assertEqual(filtered_files, expected_files) - @patch('os.walk') - @patch('os.path.getsize') - def test_skip_directories(self, mock_getsize, mock_walk): - mock_walk.return_value = [ - ('/scan_root', ['dir1', 'dir2'], ['file1.js']), - ('/scan_root/dir1', [], ['file2.js']), - ('/scan_root/dir2', [], ['file3.py']), + def test_skip_directories(self): + files = [ + 'file1.js', + 'dir1/file2.js', + 'dir2/file3.py', ] - - mock_getsize.side_effect = [100, 200, 300] + self.create_files(files) self.scan_filter.skip_patterns.append('dir2/') - expected_files = ['./file1.js', 'dir1/file2.js'] + expected_files = ['file1.js', 'dir1/file2.js'] - filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) self.assertEqual(filtered_files, expected_files) - @patch('os.walk') - @patch('os.path.getsize') - def test_custom_skip_patterns(self, mock_getsize, mock_walk): - mock_walk.return_value = [ - ('/scan_root', [], ['file1.txt', 'file2.md', 'file3.py', 'file4.rst']), + def test_custom_skip_patterns(self): + files = [ + 'file1.txt', + 'file2.md', + 'file3.py', + 'file4.rst', ] - - mock_getsize.side_effect = [100, 200, 300, 400] + self.create_files(files) self.scan_filter.skip_patterns.append('*.rst') - expected_files = ['./file3.py'] + expected_files = ['file3.py'] - filtered_files = self.scan_filter.get_filtered_files_from_folder('/scan_root') + filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) self.assertEqual(filtered_files, expected_files) From fcee407fda3694ae57560d53fb23b7e79b2ae7f8 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Dec 2024 10:34:03 +0100 Subject: [PATCH 235/489] feat: SP-1856 Fix unit tests --- tests/test_scan_filter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_scan_filter.py b/tests/test_scan_filter.py index d7cb17b2..e77ad266 100644 --- a/tests/test_scan_filter.py +++ b/tests/test_scan_filter.py @@ -23,8 +23,8 @@ def create_files(self, files): def test_default_extensions(self): files = [ - 'file1.go', 'file2.js', + 'file1.go', 'dir1/file3.py', 'dir1/file4.go', 'dir2/file5.js', @@ -35,13 +35,13 @@ def test_default_extensions(self): expected_files = [ 'file2.js', 'file1.go', - 'dir2/file5.js', 'dir1/file3.py', 'dir1/file4.go', + 'dir2/file5.js', ] filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(filtered_files, expected_files) + self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_default_folders(self): files = [ @@ -60,7 +60,7 @@ def test_default_folders(self): ] filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(filtered_files, expected_files) + self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_skip_files_by_size(self): self.scan_filter.min_size = 150 @@ -81,7 +81,7 @@ def test_skip_files_by_size(self): expected_files = ['file3.py', 'file2.go'] filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(filtered_files, expected_files) + self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_skip_directories(self): files = [ @@ -96,7 +96,7 @@ def test_skip_directories(self): expected_files = ['file1.js', 'dir1/file2.js'] filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(filtered_files, expected_files) + self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_custom_skip_patterns(self): files = [ @@ -112,7 +112,7 @@ def test_custom_skip_patterns(self): expected_files = ['file3.py'] filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(filtered_files, expected_files) + self.assertEqual(sorted(filtered_files), sorted(expected_files)) if __name__ == '__main__': From 614a3db401beba28ea7cb12b3cb9547ba7398dd2 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 9 Dec 2024 15:15:14 +0100 Subject: [PATCH 236/489] feat: SP-1856 Update scanoss settings schema --- .../_static/scanoss-settings-schema.json | 127 ++++++++++++++---- .../scanoss/data/scanoss-settings-schema.json | 127 ++++++++++++++---- src/scanoss/scanoss_settings.py | 22 ++- 3 files changed, 224 insertions(+), 52 deletions(-) rename scanoss-settings-schema.json => src/scanoss/data/scanoss-settings-schema.json (54%) diff --git a/docs/source/_static/scanoss-settings-schema.json b/docs/source/_static/scanoss-settings-schema.json index 2a75ffa4..e9e2cdc3 100644 --- a/docs/source/_static/scanoss-settings-schema.json +++ b/docs/source/_static/scanoss-settings-schema.json @@ -27,39 +27,116 @@ "properties": { "skip": { "type": "object", - "description": "Set of rules to skip files from the scan", + "description": "Set of rules to skip files from fingerprinting and scanning", "properties": { "patterns": { - "type": "array", - "description": "List of glob patterns to skip files", - "items": { - "type": "string", - "examples": [ - "path/to/folder", - "path/to/folder/**", - "path/to/folder/**/*", - "path/to/file.c", - "path/to/another/file.py", - "**/*.ts", - "**/*.json" - ] - }, - "uniqueItems": true + "type": "object", + "properties": { + "scanning": { + "type": "array", + "description": "List of glob patterns to skip files from scanning", + "items": { + "type": "string", + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + }, + "uniqueItems": true + }, + "fingerprinting": { + "type": "array", + "description": "List of glob patterns to skip files from fingerprinting", + "items": { + "type": "string", + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + }, + "uniqueItems": true + } + } }, "sizes": { "type": "object", - "description": "Set of rules to skip files based on their size", + "description": "Set of rules to skip files based on their size.", "properties": { - "min": { - "type": "integer", - "description": "Minimum size of the file in bytes" + "scanning": { + "type": "array", + "items": { + "type": "object", + "properties": { + "patterns": { + "type": "array", + "description": "List of glob patterns to apply the min/max size rule", + "items": { + "type": "string", + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + } + }, + "min": { + "type": "integer", + "description": "Minimum size of the file in bytes" + }, + "max": { + "type": "integer", + "description": "Maximum size of the file in bytes" + } + } + } }, - "max": { - "type": "integer", - "description": "Maximum size of the file in bytes" + "fingerprinting": { + "type": "array", + "items": { + "type": "object", + "properties": { + "patterns": { + "type": "array", + "description": "List of glob patterns to apply the min/max size rule", + "items": { + "type": "string" + }, + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + }, + "min": { + "type": "integer", + "description": "Minimum size of the file in bytes" + }, + "max": { + "type": "integer", + "description": "Maximum size of the file in bytes" + } + } + } } - }, - "required": ["max"] + } } } } diff --git a/scanoss-settings-schema.json b/src/scanoss/data/scanoss-settings-schema.json similarity index 54% rename from scanoss-settings-schema.json rename to src/scanoss/data/scanoss-settings-schema.json index 2a75ffa4..e9e2cdc3 100644 --- a/scanoss-settings-schema.json +++ b/src/scanoss/data/scanoss-settings-schema.json @@ -27,39 +27,116 @@ "properties": { "skip": { "type": "object", - "description": "Set of rules to skip files from the scan", + "description": "Set of rules to skip files from fingerprinting and scanning", "properties": { "patterns": { - "type": "array", - "description": "List of glob patterns to skip files", - "items": { - "type": "string", - "examples": [ - "path/to/folder", - "path/to/folder/**", - "path/to/folder/**/*", - "path/to/file.c", - "path/to/another/file.py", - "**/*.ts", - "**/*.json" - ] - }, - "uniqueItems": true + "type": "object", + "properties": { + "scanning": { + "type": "array", + "description": "List of glob patterns to skip files from scanning", + "items": { + "type": "string", + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + }, + "uniqueItems": true + }, + "fingerprinting": { + "type": "array", + "description": "List of glob patterns to skip files from fingerprinting", + "items": { + "type": "string", + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + }, + "uniqueItems": true + } + } }, "sizes": { "type": "object", - "description": "Set of rules to skip files based on their size", + "description": "Set of rules to skip files based on their size.", "properties": { - "min": { - "type": "integer", - "description": "Minimum size of the file in bytes" + "scanning": { + "type": "array", + "items": { + "type": "object", + "properties": { + "patterns": { + "type": "array", + "description": "List of glob patterns to apply the min/max size rule", + "items": { + "type": "string", + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + } + }, + "min": { + "type": "integer", + "description": "Minimum size of the file in bytes" + }, + "max": { + "type": "integer", + "description": "Maximum size of the file in bytes" + } + } + } }, - "max": { - "type": "integer", - "description": "Maximum size of the file in bytes" + "fingerprinting": { + "type": "array", + "items": { + "type": "object", + "properties": { + "patterns": { + "type": "array", + "description": "List of glob patterns to apply the min/max size rule", + "items": { + "type": "string" + }, + "examples": [ + "path/to/folder", + "path/to/folder/**", + "path/to/folder/**/*", + "path/to/file.c", + "path/to/another/file.py", + "**/*.ts", + "**/*.json" + ] + }, + "min": { + "type": "integer", + "description": "Minimum size of the file in bytes" + }, + "max": { + "type": "integer", + "description": "Maximum size of the file in bytes" + } + } + } } - }, - "required": ["max"] + } } } } diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 0eb06277..4a955385 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import List, TypedDict +import importlib_resources from jsonschema import validate from scanoss.utils.file import validate_json_file @@ -69,12 +70,29 @@ def __init__( self.settings_file_type = None self.scan_type = None - with open('scanoss-settings-schema.json') as f: - self.schema = json.load(f) + self.schema = self._load_settings_schema() if filepath: self.load_json_file(filepath) + def _load_settings_schema(self) -> dict: + """ + Load the SCANOSS settings schema from a JSON file. + + Returns: + dict: The parsed JSON content of the SCANOSS settings schema. + + Raises: + ScanossSettingsError: If there is any issue in locating, reading, or parsing the JSON file + """ + try: + schema_path = importlib_resources.files(__name__) / 'data' / 'scanoss-settings-schema.json' + with importlib_resources.as_file(schema_path) as f: + with open(f, 'r', encoding='utf-8') as file: + return json.load(file) + except Exception as e: + raise ScanossSettingsError(f'ERROR: Problem parsing Scanoss Settings Schema JSON file: {e}') from e + def load_json_file(self, filepath: str) -> 'ScanossSettings': """ Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. From 14da9412d9f3366ed3ecd0b18222f1cde13a46c8 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 9 Dec 2024 15:25:40 +0100 Subject: [PATCH 237/489] feat: SP-1856 Update docs --- docs/source/scanoss_settings_schema.rst | 269 +++++++++++++++++------- 1 file changed, 198 insertions(+), 71 deletions(-) diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index 23350c37..e24cfb26 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -32,13 +32,13 @@ The ``settings`` object allows you to configure various aspects of the scanning Skip Configuration ------------------ -The ``skip`` object lets you define rules for excluding files from being scanned. This can be useful for improving scan performance and avoiding unnecessary processing of certain files. +The ``skip`` object lets you define rules for excluding files from being scanned or fingerprinted. This can be useful for improving scan performance and avoiding unnecessary processing of certain files. Properties ~~~~~~~~~~ -skip.patterns -^^^^^^^^^^^^^ +skip.patterns.scanning +^^^^^^^^^^^^^^^^^^^^^^ A list of patterns that determine which files should be skipped during scanning. The patterns follow the same format as ``.gitignore`` files. For more information, see the `gitignore patterns documentation `_. :Type: Array of strings @@ -48,15 +48,116 @@ A list of patterns that determine which files should be skipped during scanning. { "settings": { - "skip": { - "patterns": [ - "*.log", - "!important.log", - "temp/", - "debug[0-9]*.txt", - "src/client/specific-file.js", - "src/nested/folder/" - ] + "skip": { + "patterns": { + "scanning": [ + "*.log", + "!important.log", + "temp/", + "debug[0-9]*.txt", + "src/client/specific-file.js", + "src/nested/folder/" + ] + } + } + } + } + +skip.patterns.fingerprinting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +A list of patterns that determine which files should be skipped during fingerprinting. The patterns follow the same format as ``.gitignore`` files. For more information, see the `gitignore patterns documentation `_. + +:Type: Array of strings +:Required: No +:Example: + .. code-block:: json + + { + "settings": { + "skip": { + "patterns": { + "fingerprinting": [ + "*.log", + "!important.log", + "temp/", + "debug[0-9]*.txt", + "src/client/specific-file.js", + "src/nested/folder/" + ] + } + } + } + } + +skip.sizes.scanning +^^^^^^^^^^^^^^^^^^^ +Rules for skipping files based on their size during scanning. + +:Type: Object +:Required: No +:Properties: + * ``patterns`` (array of strings): List of glob patterns to apply the min/max size rule + * ``min`` (integer): Minimum file size in bytes + * ``max`` (integer): Maximum file size in bytes (Required) +:Example: + .. code-block:: json + + { + "settings": { + "skip": { + "sizes": { + "scanning": [ + { + "patterns": [ + "*.log", + "!important.log", + "temp/", + "debug[0-9]*.txt", + "src/client/specific-file.js", + "src/nested/folder/" + ], + "min": 100, + "max": 1000000 + } + ] + } + } + } + } + +skip.sizes.fingerprinting +^^^^^^^^^^^^^^^^^^^^^^^^^ +Rules for skipping files based on their size during fingerprinting. + +:Type: Object +:Required: No +:Properties: + * ``patterns`` (array of strings): List of glob patterns to apply the min/max size rule + * ``min`` (integer): Minimum file size in bytes + * ``max`` (integer): Maximum file size in bytes (Required) +:Example: + .. code-block:: json + + { + "settings": { + "skip": { + "sizes": { + "fingerprinting": [ + { + "patterns": [ + "*.log", + "!important.log", + "temp/", + "debug[0-9]*.txt", + "src/client/specific-file.js", + "src/nested/folder/" + ], + "min": 100, + "max": 1000000 + } + ] + } + } } } @@ -90,28 +191,7 @@ Examples with Explanations # Match files like test1.js, test2.js, etc. test[0-9].js -skip.sizes -^^^^^^^^^^ -Rules for skipping files based on their size. -:Type: Object -:Required: No -:Properties: - * ``min`` (integer): Minimum file size in bytes - * ``max`` (integer): Maximum file size in bytes (Required) -:Example: - .. code-block:: json - - { - "settings": { - "skip": { - "sizes": { - "min": 100, - "max": 1000000 - } - } - } - } Complete Example ------------------- @@ -122,37 +202,60 @@ Here's a comprehensive example combining pattern and size-based skipping: { "settings": { "skip": { - "patterns": [ - "# Node.js dependencies", - "node_modules/", - - "# Build outputs", - "dist/", - "build/", - - "# Logs except important ones", - "*.log", - "!important.log", - - "# Temporary files", - "temp/", - "*.tmp", - - "# Debug files with numbers", - "debug[0-9]*.txt", - - "# All test files in any directory", - "**/*test.js" - ], + "patterns": { + "scanning": [ + "# Node.js dependencies", + "node_modules/", + + "# Build outputs", + "dist/", + "build/" + ], + "fingerprinting": [ + "# Logs except important ones", + "*.log", + "!important.log", + + "# Temporary files", + "temp/", + "*.tmp", + + "# Debug files with numbers", + "debug[0-9]*.txt", + + "# All test files in any directory", + "**/*test.js" + ] + }, "sizes": { - "min": 512, - "max": 5242880 + "scanning": [ + { + "patterns": [ + "*.log", + "!important.log" + ], + "min": 512, + "max": 5242880 + } + ], + "fingerprinting": [ + { + "patterns": [ + "temp/", + "*.tmp", + "debug[0-9]*.txt", + "src/client/specific-file.js", + "src/nested/folder/" + ], + "min": 512, + "max": 5242880 + } + ] } } } } - BOM Rules --------- @@ -278,20 +381,44 @@ Here's a complete example showing all sections: }, "settings": { "skip": { - "patterns": [ - "node_modules/", - "dist/", - "build/", - "*.log", - "!important.log", - "temp/", - "*.tmp", - "debug[0-9]*.txt", - "**/*test.js" - ], + "patterns": { + "scanning": [ + "node_modules/", + "dist/", + "build/", + ], + "fingerprinting": [ + "*.log", + "!important.log", + "temp/", + "*.tmp", + "debug[0-9]*.txt", + "**/*test.js" + ] + }, "sizes": { - "min": 512, - "max": 5242880 + "scanning": [ + { + "patterns": [ + "*.log", + "!important.log", + ], + "min": 512, + "max": 5242880 + } + ], + "fingerprinting": [ + { + "patterns": [ + "temp/", + "debug[0-9]*.txt", + "src/client/specific-file.js", + "src/nested/folder/" + ], + "min": 512, + "max": 5242880 + } + ] } } }, From 3a778a61103ac0faa0af20eea390c9c3e3eda60c Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 12 Dec 2024 12:35:34 +0100 Subject: [PATCH 238/489] feat: SP-1856 Add check for hidden files, remove unused functions --- src/scanoss/scan_filter.py | 8 ++++ src/scanoss/scanner.py | 76 -------------------------------------- 2 files changed, 8 insertions(+), 76 deletions(-) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index 46565374..e022c0a3 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -275,6 +275,10 @@ def get_filtered_files_from_files(self, files: List[str]) -> List[str]: """ filtered_files = [] for file in files: + if not self.hidden_files_folders and file.startswith('.'): + self.print_debug(f'Skipping file: {file} (hidden file)') + continue + file_path = Path(file).resolve() file_rel_path = file_path.relative_to(Path.cwd()) @@ -309,6 +313,10 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: continue for filename in filenames: + if not self.hidden_files_folders and filename.startswith('.'): + self.print_debug(f'Skipping file: {filename} (hidden file)') + continue + file_path = dirpath / filename file_rel_path = rel_path / filename file_size = file_path.stat().st_size diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index d8296c1c..9bd1c2d3 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -58,12 +58,6 @@ FAST_WINNOWING = False from .winnowing import Winnowing -FILTERED_DIRS = { # Folders to skip - "nbproject", "nbbuild", "nbdist", "__pycache__", "venv", "_yardoc", "eggs", "wheels", "htmlcov", "__pypackages__" -} -FILTERED_DIR_EXT = { # Folder endings to skip - ".egg-info" -} FILTERED_EXT = [ # File extensions to skip ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake", ".cnf", @@ -186,9 +180,6 @@ def __init__( self.post_file_count = post_size if post_size > 0 else 32 # Max number of files for any given POST (default 32) if self._skip_snippets: self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets - self.skip_extensions = FILTERED_EXT - if skip_extensions: # Append extra file extensions to skip - self.skip_extensions.extend(skip_extensions) self.scan_settings = scan_settings self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None @@ -211,73 +202,6 @@ def _maybe_set_api_sbom(self): if sbom: self.scanoss_api.set_sbom(sbom) - def __filter_files(self, files: list) -> list: - """ - Filter which files should be considered for processing - :param files: list of files to filter - :return list of filtered files - """ - file_list = [] - for f in files: - ignore = False - if f.startswith(".") and not self.hidden_files_folders: # Ignore all . files unless requested - ignore = True - if not ignore and not self.all_extensions: # Skip this check if we're allowing all extensions - f_lower = f.lower() - if f_lower in FILTERED_FILES: # Check for exact files to ignore - ignore = True - if not ignore: - for ending in self.skip_extensions: # Check for file endings to ignore (static and user supplied) - if ending and f_lower.endswith(ending): - ignore = True - break - if not ignore: - file_list.append(f) - return file_list - - def __filter_dirs(self, dirs: list) -> list: - """ - Filter which folders should be considered for processing - :param dirs: list of directories to filter - :return: list of filtered directories - """ - dir_list = [] - for d in dirs: - ignore = False - if d.startswith(".") and not self.hidden_files_folders: # Ignore all . folders unless requested - ignore = True - if not ignore and not self.all_folders: # Skip this check if we're allowing all folders - d_lower = d.lower() - if d_lower in FILTERED_DIRS: # Ignore specific folders (case insensitive) - ignore = True - elif self.skip_folders and d in self.skip_folders: # Ignore user-supplied folders (case sensitive) - ignore = True - if not ignore: - for de in FILTERED_DIR_EXT: # Ignore specific folder endings (case insensitive) - if d_lower.endswith(de): - ignore = True - break - if not ignore: - dir_list.append(d) - return dir_list - - @staticmethod - def __strip_dir(scan_dir: str, length: int, path: str) -> str: - """ - Strip the leading string from the specified path - Parameters - ---------- - scan_dir: str - Root path - length: int - length of the root path string - path: str - Path to strip - """ - if length > 0 and path.startswith(scan_dir): - path = path[length:] - return path - @staticmethod def __count_files_in_wfp_file(wfp_file: str): """ From 6a3fc5597d18ac84a456f30fdef1d06d47f266c5 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 12 Dec 2024 12:43:08 +0100 Subject: [PATCH 239/489] feat: SP-1856 Add case insensitive for dir names --- src/scanoss/scan_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index e022c0a3..5ae76b07 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -337,6 +337,6 @@ def _should_skip_dir(self, dir_rel_path: str) -> bool: is_hidden = dir_path != Path('.') and any(part.startswith('.') for part in dir_path.parts) return ( (is_hidden and not self.hidden_files_folders) - or any(dir_rel_path == p.rstrip('/') for p in self.skip_patterns) + or any(dir_rel_path.lower() == p.rstrip('/').lower() for p in self.skip_patterns) or self.path_spec.match_file(dir_rel_path + '/') ) From 5e2b515607fb801d6a7bf66ff650ecc447b1239e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 12 Dec 2024 13:06:42 +0100 Subject: [PATCH 240/489] feat: SP-1856 Add copyright to scan_filter class --- src/scanoss/scan_filter.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index 5ae76b07..04b23d99 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -1,3 +1,27 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + import os from pathlib import Path from typing import List From 86e31f2591a020188380cbc6d417fb95289f7872 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 12 Dec 2024 14:47:45 +0100 Subject: [PATCH 241/489] feat: SP-1856 Use fast winnowing --- src/scanoss/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 9bd1c2d3..9ca793a3 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -51,7 +51,7 @@ FAST_WINNOWING = False try: - from .winnowing import Winnowing + from scanoss_winnowing.winnowing import Winnowing FAST_WINNOWING = True except ModuleNotFoundError or ImportError: From 3da992981d0050a5a1ed2bc578dbec841503cd75 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 12 Dec 2024 16:14:40 +0100 Subject: [PATCH 242/489] feat: SP-1856 Make scan_filter case insensitive --- src/scanoss/scan_filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scanoss/scan_filter.py b/src/scanoss/scan_filter.py index 04b23d99..0633095e 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/scan_filter.py @@ -316,7 +316,7 @@ def get_filtered_files_from_files(self, files: List[str]) -> List[str]: self.print_debug(f'Skipping file: {file} (size: {file_size})') continue - if self.path_spec.match_file(str(file_rel_path)): + if self.path_spec.match_file(str(file_rel_path).lower()): self.print_debug(f'Skipping file: {file}') continue @@ -348,7 +348,7 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: if file_size < self.min_size or file_size > self.max_size: self.print_debug(f'Skipping file: {file_rel_path} (size: {file_size})') continue - if self.path_spec.match_file(str(file_rel_path)): + if self.path_spec.match_file(str(file_rel_path).lower()): self.print_debug(f'Skipping file: {file_rel_path}') continue else: @@ -362,5 +362,5 @@ def _should_skip_dir(self, dir_rel_path: str) -> bool: return ( (is_hidden and not self.hidden_files_folders) or any(dir_rel_path.lower() == p.rstrip('/').lower() for p in self.skip_patterns) - or self.path_spec.match_file(dir_rel_path + '/') + or self.path_spec.match_file(dir_rel_path.lower() + '/') ) From 9175149d0a1bcbe0144dbbf48f8f2f2472e8bf6f Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz Date: Thu, 12 Dec 2024 15:20:07 -0300 Subject: [PATCH 243/489] updated documentation --- docs/source/conf.py | 4 ++-- docs/source/scanosslogo.jpg | Bin 0 -> 138601 bytes docs/source/scanosslogo.png | Bin 31268 -> 0 bytes 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 docs/source/scanosslogo.jpg delete mode 100644 docs/source/scanosslogo.png diff --git a/docs/source/conf.py b/docs/source/conf.py index 9d25efb0..bb539e8d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,7 +6,7 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'Documentation for scanoss-py' +project = 'scanoss-py' copyright = '2024, Scan Open Source Solutions SL' author = 'Scan Open Source Solutions SL' @@ -24,5 +24,5 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = 'furo' -html_logo = 'scanosslogo.png' +html_logo = 'scanosslogo.jpg' html_static_path = ['_static'] diff --git a/docs/source/scanosslogo.jpg b/docs/source/scanosslogo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7e6708d35af14e3bc8772f6b499ba3853b01d425 GIT binary patch literal 138601 zcmeFZcf8}o^(gMUy}`2VvUIpCY{TrzmMvRe*agW=#j;gw%QguhOSWuFmTg(GCDRfh zl!ZX(H4u79fP}J?1PDEpP?p{j2%#l}KodGCE*{)q19+DD_AqcdmD zoH=F28=l_qhdBi@8r`4eGYK*k_l%8sJadRt-DpkP!sBTgmdPja@j|gQKK&ujB+pFG zG>_lok>qZh5fWJsxX5^PybCH#z+YbqDwF!9AlnK}b1Oj-Q30NJ$dkpK0;C&`wrD-Uj&IHYErKwE7 zcLDh5o?QX(j=Bk4-UPNP4xrQHiMHCVSyyY;_%eC$B9Z`uUOU!R+2TzRK}yNW?M2` zk7v~9iB_I1CwMy!m>lv)LZKD#O2GcFu7COPt6KjWm>Anpjd!`>n2jxb@awX#+kRcv zy1?T>-v_+;(br|=Gd-Rge&F$J^X%7UyZyxDnR|rCv+m(d`m=CCUTT)rUIjt@et%`X zBCnia=vUXbR`{ysKL<8>-^z*ieZ{-+Sfwi2x-~w*RJozuj;X zvNoY(557{Zm=y!ODi5TrZm3|m4W(|?TgG_Z_-7;hZ#LV6h6!*a*8rG*(HEWt`>*tD z@$drA1y9Dl8n@KUyk}zOi95eGl6wGy=flsx`J zQ9e5^+h%V926JNKW2$GCXRc>~XIsxso?Si5JS#k~C+ykNvyUh4p*$H+)>HKC=aD@% zPt#+1`ksS4-}fBpIo9(d iXJm+~X@?7q@%5#n92G6fNw|nmKJm7i6^MvO)&x@W{ zJb(7Q?fJm->DbuV^s&vx=8tVVw)5EHG4B{WhK{Wni;tzo__5;H0b}ZzF=me)G1UJHO}c&31CySZ^x~vHPkMi{XY%aH3n%X~dBtRO za&&TLa%pmPvN`#X$!jN{Hu-|dS53Zo^1YKEoBZPBHz$8QW!jYaQ^u#Pn6l>-VoGj` zJf$<`kSWJaIcv&gQ`SwnYszC&UYhdulrN@kHg$)o-l?mml2Z#))v5idM@~I$>LpXJ zpL+MyC#JqK^@C~CrY)SdbXs^CF|9bQG3~%<$4&e3w5zB6dfKDYUY_>;^y$;Lo$j5! zdU|HMG~JqhqaoCL0W?V7j)*0((yf)+0 znOn?UJag5|%uHpbJM)B@7tOqR=EE~znfd9gd9#+zT0JW}t1;{Qv(A`x)vUW`JwNN+ z*|TRanjM+V&92Y>-t04G|9tjuX1_H1<2hT-*?mrQ&Ufb6bB>>L>6~BBd1lVLn{Bq) zlFjznthAZ6+3}lQzS*6d{cf|5HlM%w%FW5mmCX;`{OrxwZN7fZXs=P$QEaBanlx0ZSnrR`SbkqGV_}Aj+=MIy!+<8zUA~S zmuwl|QrYr|Eic^i&MjY_KY9MH^P}_S`G?QHX#U;vUs*71!O{ifg2sYl7hJjEp#^Vm zwdGd9t@2wrTb;etEnB^~aPq<>3(1B0!V?x=v+&7={oyv- zZ?n%fm2K8;bM-b)Z1c&s$Fv5URS`MW%{%cqN$F3K%BY|+(= zUf6ZkuIR4ST~FEdj$Pm0ZRg#X-45FAr@KA7c;;esac%MGi|<|h;gY3G#3e^9xnarc zOSf4{E*&iW>C)$yZMJOhWu0XgEPH(UwB_h>ZTXLvKeGFz-I3jEyPv)L!`?~Wkhkvr zvG>tErtT5h!`S13J)T-IX9c#Rx8kQOURt?uWoqSND{ol&7PK2wf=-0)g}(41KHYbb z?>YaLe#(EC|7QQ+;N4*vJ{x{Kuvs7xI5co$;GLj1SP7mRd>WaLFvwBJ9mwaQaL5c@ z6?!AQTlj$RIpL?!1t^OihyErqHG)SDjr=O|@v6`&Yt=QY{<`OiJ+(cr*z@(hmhM&A z>!Q70UcG3wwEFzjFRmG1bHJLPta)+oUG|ptzF_Y^?6d1W%08Fu^BT4sYhXXa-okyj ziC>R@7+n=TIQr|@35y)-nZX&`|11Lvj5!urTwoxVCn(l z0aqOGnUs+(klrto<#Wn!%Tf7s`OnHe$|=elmDQD#Dz8;nS5Kzq&c2c|h|fV@pFb?rQDSa$1kJz3n60FLqXSPVKyH z(&i=IvF>-fH(CpTG4-Im%>IG>Qg2P~?A}NHeE(W!o?|!0s>O^A8z2qE)A|lh>Uxe#+WY-aB>w zQ|~)1c-ncV&pO>c{lzm;_pVWWy^!ds2*IlsWg3~UXc46g-##y5)phXWe?lts8C|-1h#j+rNJEcJ=m`@7Vv2=k6@r`S@MjT@T$&-~F3=68GGB zFLv*3zghj8U){IrzMJkx@4w-J&;#opL>|2Ux5#g=e<<|Ox`)FL-}p%6k((dg>(N`+ z@4f!^$6}A&^EmnV15c!%c=XBKlTSbOou^)WT6y~QXY^;@dA9fLr_X)w`KiyZePN3i zPW#;szq{b~%YT3Mi=h{9c`5eNgMSeI@Z8Jt%YXi3_m7{wa>T2fy?WYfJHK}M>+tKp zdL!}1BY)cWPp|yB{pU~LJn}E|{&LP+%ig-~ulxMrkbH?J z2D}-w-X2_5-wtO$>n{B@C``73yG0Cc|nHZZDzQ_t6lJ$Xs@vTC_D}2FKKHnxtMm4{I zBU`hZWkag#-Qf^^#o6IlV!HMBr4t>Q7!18WKJnF-%x)#?wkxX_@s%!c5#-9Ief}C5 z=*R@KSM}?PRa*lip&?nD6n#zA|JtZlnQ#WzprgJF8*ME5SFNfovng3?C^nw_`ehpn z|5dwsQ&KByphyD>Vi+fAC5l%u98=?%fZ-&D#h@^Ub#uvDPf11wQZmvfV~ovUSdi27 zH0JeT!wbhi{fR4%E!ow<_Qbe}#^B;G9P*Nk7}sN*Kf>j-NK(i-2~ZI)-}lLM|U8iL}7`M(-OI ze)++c#R-ZgE}!csb5Vc}TpJNifdAIT7X!GfOf=08+Yg2aOUeA8*arp03Ci%U65#v8 zO4VVdT(N15?$sgXIa#lj#0h$8L!xpe|Hk`z|0@b}#V;#qBP6Ay-!4sn!|vmKaIX#T zyAw#Gq+;at;k|Eyl4w$C>be5JQI{jbH)~}gZ%Gp%G;~;1EnkxoJuE|HW`Z}nXIY` z$$X%k%=?t6pBo|sqiYrmlt#wWy>bDsl?~AEB(KXbkhqOk+!&6wIx+I;MjS~uiWXB3 zI$u8D_&uUf_7CNW;&TklWHmTl_qnOYNF=_|;9GA;JlwQe0*qMw%YxV-2ZVii$Cnu~ zp-cZjBEjR4a8K0x3Sl_}s>T|AP!`kv4k=!5f`PJvInb^l{T;4SZPLh*m13 z8($#=WuO6=%Q;4s2;L{hTD??22`DboE{Tj?goh(PtoE7K1RSw{_BAioG0zSxTO7zRO{$EA- zfsqPUFGf>+GB~18i${e%mBVyjwA7~@%xFbx7@fh|YP6-&UCb;-`&=r8S-EHs zOPN?N2XrLPU>5ECcR7|v0(T~fygr6(uc(s_s)uM>k1iBk;al!rd;xate#XaYDf=NgUA#6RSXYgd`BI8jNCHlIW{$G+rb{wPr_2 zlGN3DXNWMG3X(pxVMN(36~{C^nr%_pzOF?J8rA4GwP?9T4f=*2t#;`+*D|6FADzeA zooFjgH?fW#?dIqJW)7nLZfb~-RJ%yjA5Te~jYSbP)x~;b)EiHExxNsKwivGO$T1?v zG*s7!F&G1LgF%e*r76{q#qzDRg29=Xq^0})KxK$9R#h|ce$b9-TBd{{P^{g}SZY{^ z*+DkSp?b_0XA5d16hn$^w-2!~Z=Z|uSTi1HxFU~x<5ZWk`B)*&=J+APxC8!jC|~Bm z&`B-c<%w2Yl?AFsCE`s^sNr-Y-erWK!^Gk)ljR(+kOrA-8)p-7Z!U+|I3p3)a#Bqg zbvZ%%a)Vm7pWw-S8qbS~0?<&77)ewx5y)CIp=n|nm)L|!ivC(TlyH0nj#uV$!K$$z-lLq_AGA_mb&c(Oc8hWG+{doF<y zxN^p^8p&X*+^zK_0wZO*<_HAQm0Qk$86qUuoI=$61W~}0CLf4_6iOvgLolM*s_1yg zOW0a9?x0=*)~Y)0g$Qp;O$aznCi`kL8jF)GRwJSbk}TwELxe4X$dJ`;opPx>*&^$v zn^MSrtC5MOnUX z3H?MWrgeu1%XPP&O3PL(>c>-sqSbK&wN$NWXQQEL%9QPJ9o16)R!i>2z_h!rKgOgPUvW?!WP%J7_t67hPg6l39Lr_XIiJ5a zC`8j$ADkYP+;k@hhqAJkhOvMVs}?di7tF=#OomaBA%%EWk7f!e(u)~Lrcn$jv5uRu z(QqnjnVC>49E$Y~Hkm`)u|b0s$Vf2@*Vqc?C1Z%nb}(-+hSpd(4z(IkeTa~Yrm<2y z+T=0{PUjLGPIB5@F+^DLRe|G@Ck0=f7t#tvU7Uy}13U5xmZcpjk&LkfclwI>q(qU+ZrQPfLzqD6o&sjZuS z(U&jl`JP^g<>i__5DS8$R1yKIP-|DpiBP1_k5r3!@1Tg~)jS*>>BA!H)Ut50UaaQp zf=J879^T;KOrhwtG!8gtrHrn#q9~LqN|O~Op=8$#R;*-8UcJS_by3PVZ4PcqQZ+F` zC?q>osi&A(*fu06(#^weuguk~ViHEmHOiLYaJ=jwy-E_|<*3!K88KDPwjI4d^kvO* z+XjQne*eHGxZJSI3W56uM6RV2^Zp>9OKDrMmrz6{oDRkdHM_z%2yHZ}N<|6rMn|vo z`EW7Ni&movR4oi@)m$RdDg^OrlkyG`_6@XA4e}6D#0izI*+lxXK{n+G;@1p?u5eVyHXOKGN1#wE z*;HXVF%s-nS*lYVT5R>DhSFi1?S!N`pZ@oVTY`8cZARmRZz0*mV3MyX@rAxuO)b)a3s`& z3Yg5qx_z;V2T`ipw|X&=VL{M(5KH(IbbD4|LlhI_rTn(ik8q^ZQyuo$QfDlH3b2SSjVwxf|?E?XO9sBF;c zHHu!=L#zqKLh6izg1j&;QmyajoaJ()T<2k0Ctl^I6aCH{XSwxL-N_8~KyJ|14a|1rB zb!fL3V58|AXK10!AYD^N`mjV~V?LWCsTM;;Yig-R0oHW{U*1aRY20*!5F7Vrvc6Jb zB0wPe9i^a0#1dQsliliquv)Je;QflB1S_L8r5&ykes499iPwq}Q74;pJ?H{K_eM2D z$m{J&v03Vr4GOLp2;FGYVzV8pw>urXYf7PBYl6R^`(zB~a`9o8ftbXjKq|@;46emv zcyz+Vh$nJ1ZNP)#YA%K*ji&T9bsFE7prZk>CM#6t_uN zsJQAVIjye_yG%7oiDX^TlteCob~1y$tG0b!c2J`P&0?}SgynSw_^R!4A>pqS{5){Y zm6jy7&GN{t@Y`khOCJ2w?#FZrEU`$v)NI?bQ3_SHPDX1>lF^}?oleNL%#ODQNuyE5 z`an#uXnaD>Vkj2L1v}wHq4@873=#hGws|^ihsA-_E_cFBAz-!gsX^B5#`#1NDS`Mc zmZW<)#2Qd0-)z9Vw=-JxvZjdb>hIxp}V<3oGAK(P_~(mW#cFe_Sqcp!n&&7wO9fL(F?o=^)Veb*pWAi$mulR$y9`PI{;;JJ){Mz#Vpi5r5j@DSh)6UWs6&O?2nO||a*L&Hv)`7ylAf%Ej9e-d8rj%LD#HOQH;I*6Rw&GN zac@9RW(Ai>$?0yIiw)okjH&fjs8qMPZ(;7Y@NhUV6O1&afmJf?kS-=+%O{DI(tywL zxLaycArQOmc``CGom*RjLH^`=s_!6aS)r;`gt3; z!90ru2)D$5sP3rC7KKQ5DH;-5y^H`G8f^{|A`xz?l)^=&aKldOMqd_#NLS7jYA{Ph zswK0KQR-$l|Ba6!!f%xQ)7wfE2zAa|oltT(!dU)THe`1S=})bNU2x#9)fml~Zm*)cj!2TJe4`pTS+k=q;xZ%?e>QhCi2BRq4G6Y%b=-Vyw~ZW7-^#6 zDq7~)7HpwpkFh$vwAd)r2)R!ua4jb{gj}+h=CDjCoR0=GZ?V9FzjMkm_MyQb#yB!)YUV7Hbd0^t~AW=j<^S1C%2H`C}w z{Lqj>#2KK$#vtY`B*2_?d#O&Q&Xkj(l&_P+Q(`!3gxPkS>IHI0Jya)RsesMbbXhEW z6-&{5F}G@p7FQGPG&}N_+u?cuMCdf8!)s2hA8pfmDBqGhz==+44X#QQ^s<{ryij^j zP_n(EX=i}VOE!BZ?A1#}AA=NI^;$wJV(4g133fUthr@$36YR(8bxIROb7JDRpk@i{ z>oq?s=z3Pk5^_PX3NY)JtES5$O}dJw0t3`e#Rf4X2c>+xcVuvu`+;=0-M6W5CnS~3 zx)x0sg-8%`;i1@u0^QCgR3&3QdG;hCqPJ0OF;Q z0Yqip9^yx>k-?b{8(B83!)34_g-a$i$oSn39BA{M4m}8WT8h}Ri?*06CHsM3P>U&5 z6f;bu-t-ZuX_alW!WY9f(hyv06od6kS|C-Yvkk#;qLfjrrifS)!R2;`swN?=H(-@Y z9!iT%2`Yo-xtNXe8q{n6$Bpt6a0@Ph)KC!32}D#KQ5c6Iu8`NGuu#jq zJ&Lq@Ngg5cRnf>of&&eDAd+4lC}uRIv;&=3wwH_NjZ(YoGc~kQ_J#CLIM%`=)Sw-~ zTTI1pyp-lOn^mfoDQQCrr6}11<1}axl9eGWipT>Cs6-JFP6Gj~4o*YiR9xv5qf)2f z5+Oet?*x!yrDA1$W zuMM!C3p`~PMCx1us0_v*mOw(#SIbCoj@wYR+&^cCG-tC zheW-}PN~$6mzk`uff_*1bfj`QR3kuCl`jaXR9pk)ZbM~EqLvH>a8!ny1>gxbMxInr z6Ku+Cdo42=*F;0CRrNd)@D)TeiPk~bly4Yy2}xKi*=9I7(dS$qRr5r)*GzP5zs2e* z-#0*ztw^xQkiu>-*v8szuTp5*k!~?n=2JA^$|^0N6(DU#qT3v=s*xm{(j!E#$-+dN zM5v@(tH#_&IF+WYX0lR%+!pGFh6s^1lkA8-zX`IR1nHO=RISvz`3~1DH{^yhC`%2? z0CTYz7brq0gy2qw)I>Mt0#VUHwl5406reG%L50wqn$MPf;sQX6tVL&B(7L|qu z=>gqGn+ z5bE!_UfWbiFV&3IeOfaY6<|9U@kt%DQ}&s|L2*JINkKb=%ydt$&#=Gx7yR&&yEDi^6ip`;H= z8BhlfIgm?PU@*JjbOKPH6*|d{Jo;t;Y54{@$mRV3F&*!bVb=v#btaPu*2rR_)D8xX zT*P0*-DED@=xK5>!dc89RM4vs7OMmCWlgA=^pa3zG|HkDk6MA~Kwt*3aFEyG7E77^ zM$>DcAjJUFJ>7&b%Jre{VBjrTDb#ibwBIO!r3Pu~B$aS7VUrJ))jBou=t(@B4kJye zSQdjIpyJaCNCVQzW)0E{2#*e6nTn&jZG@C^&|9EU!t67yQ37))N!d+^pn|C7lEs1J zkB<$`E6w?MR$O~4Xp&WvBf^rLtDHuK|(eh)FY`kn_dtlv_E2U}*?)51ul=CHwj@=BT zur`s=x{TcQv7}O=G$z|Hvq&R_lUY0_4iN(XI}r*%15HOX-(ZmPijbZaBWW!LTVXkw zqm#fjEVXDWLG@D@n*=s-9?s;XVv=#Afzm+c#QcN@kV0D{*9@YYm14cs=$UPojffcr zDRio4Daf{k0IHc1T^=+jGT8C=N-jt*^@}}_dVy46wX?8S0*S;fYo`h-(o@ipg2i29 z5J4iE7g$GzoP~o?HYUT}kdFvg+l~;21Q?hyty-mKdK;-o3fSHj%MOCwb{^PK;jo!V zMNzC>D76A3gyo3kP$1Ulu%%G3BUkJ}Dw7e?s0A5pD_?8n8^r=hqDLzOwPCqFvY{m7 z0*?8`3RAZ-I#cUZ`e+x`+L$wnWAjufQdOH7u+a#Zk0@NF?QW>9tA+{7z80jgN;FNv zl@z4F)Hs;FJp1 zOnQ*PoKO|SR;wh^?UlkLtrO-zLW4-6ilm`*fL93A7Aws{q%HSYsgsKgM_EV|VO~e# zW~A+7xnQ~gQ<_^U3H72cTZup&5zwA%# z!r_tQgIl)J^?O@k0(2o+YN?7J>XboxE1ONr{%q09SG{rvupS6E5R{VS@1@gfvkHOy zfJhq<$Tf2VvJcD^cVxi^>;QsQiou8s(E%YE6U*6r#wbJ^)w+B9oL4rg10F2hNIK&;c`MVcq3;Ju0U8v18OP4sc_|FGxTMmQUqmfYMoS5;j)VB|XzC^O7H? zT>?U!gii`;s4F3+5=~X$u76{sJ)dq?g1RWGY`vuhycIFj5&N-%nnY3|kin4KDhU5* zDR0hh1tLy0P*G{WUlG#jj9W;9G^{dki!~C_c}W||VIsja8BKM97}}fgarG)m=slP7 z+94d|4y=GQa7bU(%w{R3Ys%4fJnY6nN(<{$V6lkSs6bIrX|9Thp;5wGmIe^U1pQvw zjK7GC2#C=s# zO^h6HtBqx7HIj9jCClz-h_uT$kY2}N6kBv%$KiV^I@oA5@KT_t(j7HVQ7jir6#7}j z=dXlOAFxu)WE<|d(rCYpKU^8GC8rh*Tjdt;$Ix7W6cY);O_tDf%cfFtw}TF<&9qlx zn^`W^;FNq9LDGy}OH##FBokqLbxq`HMjI&yvgO9w8tJI9BCs?yKAU2_vM}N2$B??e z0F3c^-L*kp%SSU@EQF@oNlp#xtw5(*l9R2Zzuh-l8kmAHeY6t;m~IoUPPk-&s1XBP z9lhNM0gX`Z71%U^NN^SSI~tSv?MnF=S7}Vo7k;f}As$0o0 zlOULGqm%ZAf!l+jStB2@i-Qzy`9`s?Dx3gZP{I)-X^Y+}%lIOOX&@aUWYGi&)<&$d zQqsv%)``n0ri}*@`7{Hv&pepbAkWelAfXTk(<=+pu{MM2i{|sklzb7#T~h)2Si> zn1^zGlxAhLh8TqHjoHHJdJUjjknFAfjDzqs4&P<$ZLQX4=5;2$~69Z)g;sj%P#=KP^Up`2oft z1*i*I1t{ytz!b@68OST9H;PTh>$0IHQRBl#gD?VondPc!D^LtoL@`jO1+F6qxe4mf zMh{IZoiO4Hl64SJ#GK=KNaf<0QK7{42=q+Fx}7E376 zPN=qyMO_r^{^b1xZj+E6j-?o_T;$VisRzs~;IE-!61F8KBpOX2!v-5%#i(@ZxzRoc zf$x}3wMciHUf6+wt%lhDOY#oXA>lYutYMDUYll&*>ITt? zkb&sUhnp3z-J()fL(TwH^CYzGksOvCYm89bt9xar~SaXm9`evk*>tMF3s9bQb>Lp5{We}M=YwS^m#U=uA*Wz>fURNrNCYpHYc;W!0in(u zSFd{;tlI!f?nLwj5i)gml;kS*%FUor0nw&>&Ll$JI$7f}LCEKLHP3>;slv2%+J&(k z8b>F*)=WBp)Fleh(|&}G75w$CVTuWVLUv$lNTI(R_UA=L(|CmP#afEj4rEhcOJFiA z78N%Dj3FEv=y}NXOCbJ_J2vo*!!*h!@Jcf4O9?%?fVd8%wUPS9xtt5wW?)e6nRuYz^p)Ks9&?oDc{zD#QY9LTAx_FI_b(ustXy z^h~RsqLcAKRZK~#NU-51Y7Wpz<&Ei&aEXX%STZDQQ6b_-n{Xn~4ghr=VS6Mb^ipg# zEC%dirxQpDG@er`g4U6kOe@W_GzIp_MvXv_sunZhkqJ5 zyx4XesG<(Q-mG2$iFdJd6Q@`ZKMi*KSj);}iBgy4fo@c*G|^1CZ6sb6Mz&4WVmoDM}r`=-JO8cM~8EROK z*hn8b{#t;wTd@I>BI1G)Xc34WDk1$|I5{Bnrck1S2-r?arWzsJ$(dG85B9lO7-?r* z)CR&BwB)qHm%)}>5*`jprAll<^hSCko6_3`^h&HZ2N|)PRc`2LpSN(y4s;5NpGp&H zI+FIXeLF0V0+OLTn>B%Ovv5zXHO#~yl%p}PQw-L#cEmT(3|i)*G$9j-f@^>%Ba{LC z38gEAWY3!m=fUn7B$KlSMLwpFwp>a?CxGKNT99*rLa$3B)fg49mX2|6i0bO=y?T{CADgr*Lf$QJGEG)*aIdxh&nW|sxK`S#BQMwj_M=Zrqg8$Rh=S} za#^8yBg8N=MaQ&QA7~N={Aa3z5)uZ#mZ2$;j4!8>&4!4YO{x#F3qA<0iZX-Sc?@hD ziH$4=aAGwsHf6T}!qW+$Su#L@R>VYzWvYhg@+DGr;bfx#woLd^cv3(#+6KAJydE#7 zazxOo<9el#uJp@Hg&iq}f-l=-0vRU{)As+yM=#w=2Zmw;My-lSl@gUO(sj5IAk&4W zujRl&IoIY}uBH`wU{8AlZB7Jf;t2#&fIU-%%h|5qK-yZ(8bsJsp_GnF9MXot#jT+A3=u9e|*jVeX4Sf;DG|1@Lz_Ya0${$>XD@2mX( zdpbn;PZ3n1($`1VM(5JLd2($pXmlL!?~mz?O8@@!-sm9UKRoU?JWR;6lzP?u=3zoV zJUUD`B4pJjmA&qH2?Koufme!ybTBqu|6U^wp8o zzwggiX9E8Z17G*y|1jcB(f#)&n{wk@;s3JhTUoSeh5xc^Q*L}K{9kr`D~mR*@LzUq z%8hS@|I4m#WznV;{uR4sef^^s6$AXzMIZd6#fBF>TTlry;TfAWHa0#s?(uB+i)RnE z)|&XagjRbzu+ry2agv$*7x0r>9?uqI?+wp+jsSQ6<7?8_*W|CRjjaI% z;3@e3>cRgM*zf}Q%`bVwTb|8_KLO!Uemz0uhF^Q;&6qrG^1R8Od7iO(W0U5MZMbXf zf+4w{v8iKU{;Jq)&#Xx^$0kpkK4a<>P&j$?|K-yu(`QVZI%)E(nPaoJ06#4@W%A@1 zQ>IRuI=2a40k}4K%G7D|rtjjPfpIgpoG*30f8yJ-&i}=u3$|Kx`%4So*=E<>V0`Os z7cYtA3#N1Osn^}`_#d`gx@`IGfglnJqpQH7utbs|DIV<96pN*@tW>IMt={NbcCYUa z4m6Ft>KjX}^F1YZbi!ZtKvdgcy=GyDm-FVZ@cief`-S^!4oBQrx z|JV~xKK1l7&%XS}S6+SX^*8?Xw|C!r|AP-d`uLN1o*_eL&X_)Z+8lKw$@2isyZEPX zIUnOD2wt!VE}Q57;`Xg@{-g`;cy!^e(e7c_yd4M$M_jmm#!MtDTZf-~=VLFuGi&Q@ zw%u;G#Y-mW4~Egms*S|w@*?>CwvChnx*c~gqWY9mhg5H*`P%EQUv~o_dLzk?k0}1b z|3dPHXFYQ!1J=%)G|#iQ=l8#!J)+z^^Wwd?-Pw0cb=ItTM;v_9>t|f{ zgLQXqwsdgH1&elmQ(k^wV);#L7GJl)vv}Fvt&7&LeOZ0^!Q?C3Jw1QMC%t#J*Z*?n za_4F5=1X^b$9dwYcaY7_=>O!v3oc#!*wgpD`=|P!FOD5u*>>4+H|%)JL2FJ>Z@g{& znz{2AKXCi~FI>6rL$3Yq^QUk2!L88Tt?#=d@%b^ofAElPuDWH>A;e&-)}05hJ>jP_ zk688b+QZKGoxSbuyRE#By6M66+QWWuNqJtSar7(dA=6vNk*|GH+;@NTue-jSKI`4q z)vxX~c(!}t0^^sL9=!POZJ3WgT=(vICtmSaYJ=zG{CWHS?6LPkPn`Pf`r48Ul!CSo zb@5v#onw2sU*^kC?CsNk^qg|_gUio6>ybQkTJ8PIjeRe&k;l(nxMSetL#J)#Zt!?t zdE$ko>Pd;nlTLk-Fwd z;cmH%R6B1Rvo2Blk$KRS_21mTd*}Ks_s!GA<cQjg+wCdznwxFt);&*r>GO4~4n6+#-#_ynZLj*|!yA`D_x$j) z*rD@wI%7%jq>~?h`n=hX9|j{Aqc`Mtsh$MCwq@2j_K3sbM;<)>`MJ?g_l8fH{?UQ? z4IcW0-=A>xyTMTxXgWuc+%Ko`{yE)T@1@=m=iITX@#!%qKlIWZ zbH>r{uRQ*>yKeZrW$wQ4ubp>qTYvf=?>hKcoBHstKWA@vtn|@wAAffGWAjhB@uCYZ z_~U11+`h}Q=E~T8y>pK|^YVKK$a}}e7WBrbnWsJe&eF3^KDvG1?t5PL+rFrV!A>B{()Ykv9xb?-A<-gJ3* z);U-B{&vM~mpy)KC)7nhL7ykbPg1JHO=q6E_O8ZJg1OhD2VZ#Xoe$i9+{>@?iywP| zUU6vSCGD!Cwtl0tcZ&PnsfRAviqbD}mpt2_%RJOsb@R2eE9^kc*8F8D@@Fq} zu0Q$6>(1Ju@#^@;_fbDwzISNdD`#cWpKX7~<9|~ZzyHWFuk$ZJ$=w}oe&+te4&67h z^Us$-#I7fKw{U;>>*m4Zm+rdYy3mcQ-ksYqk9s}(+MZ`x%N^BKKT&&iSsXAwPOF<@4g4R=eU<&>(CD!^5&ff-~7Sy zdnO;*n%%qcnQLx6_SoWe4_s*bAe_U0kKf6N_Zv z?O#L|+K2!0Zpj~9|MIDki=WfnhqbvsRzA9IeaJgE^wFtA_?ZLGyMn#$%H^d6r=DM5 ze0~k)v!`yg{-VpSqNtvQ932eBG@VIrzP|QQec(*|%;x`G~il z{_Syj1{)fxX+R8$9c=<2yC9b4ncoo~Lq z=UMg5m#n{g73w(ae@|Z?J?NLp@$|MgFN`1ctM>g@y3Q|tckj9bpL%AYqu%p-`s(`q zC;Hn5?ydjw@PaVsjawfObn3D{J-Gk5@a-p^bk}!w<;!=Rx!Z1s9Q*2Ro9%Q*>f=jJ zJ+XVjRkyz-p7z+a_x?f49rPpal3#uLp>gE#`)+-6URqt)WA*OZY0PgwzG3a>%uc(# zbHeEto_f@FpE&MAPhEfgygm9CEvY!+ZeD)K<>8&~I$quCl2yrLpV%Y&(oV$U+4kMb-Dh5gws_~l zZQdCKUY=99`yA@7kG>c@|50z3r))-BXk3}NVTX@$Wbn+rA1*!g>Op@#^R=sPS+eu= zBkox7*1}U)aeq8Q-WgARfB&u5-gV0dOD?8PIE^>oTX%4+_ttIu9d+qWllWbO1N-&w zetPSn?T>eU<>+f)GOoH{&;1Xnyt(rm@6_J=>%)hXo_p%?t@plX@Xo^r|8mddr`^wf z@y4y6m)>9a#qI-E&6n=_y;r-Jzp4D>vbE3L_}(s5?vkzW-OO`h2cAR0 z$P()Z+wQy9+UM^HbbmYlo+Y22eb{9my>UNxL(F*T?eqyVFTI&N;)d7c9~}0F%eT4z z^Og4>I^%DD`TiHD|8DsMPyK(Ky>(n0>)JJ3*;QyMP@qU*h zBKy#N^K@F_*R6Y;b)P>4`7tm(mx4MM!wG_V6=@v0Wa2SGpXKeOvAm3YOJ!4E=b7E& z7dhS>sE1@UM;y!^r^yhx*eKXk5Hbsw=s3d9c<1Kk7<+GST`20tj}PRtJ#MaVwM|0~ zhDbj8yWK`SktSy$Y=)VspjR-YmBaBdVQ{&s5es+EfThm6E>6BTaF2i$9ktI5`8Z^7 zUd?)+!7SE&YDF&1=Yg{xRlAOWxf+qmzIrGc6L9WQ9VsXz8F@xoN%%=X=H4Rh3*#qZ zwv~%9-uxw~g3-qmx$^4q#;M3WHKNdk7%T&fJI1cMK37gYsIw;3%pjs?!{Nh*fW`1q zswu^YbLmh-$D@cJ#_M}E9K##9Wxv>Zo0&Q+8dxp+-&aa^|9@0AM9ov`j)^wxWmd-Y z)nSurPc1@{pHlM3w{%j59>=qZOtqavnFb823%t>8k(3oapb8H9Y(}z_t%i~(&$$$~ zi!|46OutK{>O5CneL{Pq8L8^FA{N#n-pyNIjXWgJllS-Y1M8mm~&KT(PiFF zq8RRKnxdg1ry%*tJ+P8#;`vrMF-Wx6j5$mo(@f%D5;g40&CKQMS z3%dwYl*k7Q^@7GK<#b32v{AV$L@3oK-u&GEOhIa1?otP_fM)fw6d0OV@HBiCnbINC z$1+$JBb?e!zt(vk*37&%zBXRtkLp;+W?0bNl2U;7=MPsE%j?9NnVA{qhlCkrK9AMR ziEnjsbc}I~u~Rk>zJV0(b~di&`TRJLZ4GOHu3%LTfXpC*xlDP$=jIaWNOAS`KFR6m zCIA)AG%?$)dQv73xs2tIsTym{j917gJsw%OJg>b^*E8UFw4f56X+XQoP8CVu7L8`o z-MCW%9e&+Xxgk$8wKn&Hm5+6HT=+(6uh$2p9G%RzTubzsj?_!gef)JxIXqHwlhgZR z#dOT@K%!lGti^$DRoHbGH#R9TjNLoh;AVozo*RK?OcQV|JU7#4>9w8>i0tbci=&ed z^~&Xo*AhJxZg!Wu=zL!^M3C#U`+D{s{3d$=8>Q;v4|{`MKGq{rA67S2VMp+&CP&An zDWKMm%9*R82Z^~;_d$W{8Kj~;RonlK42FO`F9 zz<%BGFF=%=g}Qz?syeYPo{sMnto|ot^G;p_`Uw%4zF~tS-0W|mY>liuHFlD3>l-_1 z2~<9+FTMokXK+^^#+@vdROof_aXcFC=ERlK!3BO`GsWTdX*GmzgrjN^g$*>hOBfLJ~{}b zfu7~{UlVrUAzJ{cYKm%3#6wd|$%Mpuh%`en6S9py3UQ0zydEO%e%^-Bj7JGtNDY{Y zfp?wJHrsKIPb#hU`hTgy`2PCuvF+ek~x5sp1qp?eQrD#)afe)$Rqz3zi;wgy+0 zn7&jk$89RhptuS|?t|D21O>`hKzchWwWIKeDqXJ#@-NG~1zxQ~1{zFb^lfIz9&sN_ zmH$X*DIY0p48&N@Z5ZK}m%FORrkiwgVft70%n|Lz|D?W7=FQhItj4|?ba&%+eA0?a zNKPKjc%iTlaZU2$4JNjc5EXVp_OHCHj-z_o-(ATndr+^aI$uUGHL{zb?(zl}GT0=n zx1wW^Zl)Dl_|JgPXolAm$jNOHw&_Isf+5Su*eHvUk%2BWEEF-GpI?)p>%+oAe_TA4 zWkly8@3K1UVwQ`Nu~Lm#aeL_~$H`O_)l-jlt4jYtcF^OBh~Me$^)B(2U!9v->bUT> z9oLzYoU0pM#NhMX&lEF?6*V7~j*E2vb<55-5J|e|IFBUIEz+dhFhA34*S@-P!L|CL zCEB%J3?fx70z2{00#n%0+-Iyh>OYS>I*Z!j7CzAdJ^>)5h&bII!w%kLtYnW#ihIXn zAdJ@%Ekl#EyLZs8roQh1MA>UhspyUTG`k{s4dE*8;OY^j^$|x%UIUEDy!GnT%=lWO z!>9_qd$PIl8^{PIUH4}4B(F!k4R~CZ^t3Oo#CGt*9P!P1m2+-dJ9PRY&yD6vOQ+yB z8S`*-eaD*vi58sXV#1SauX2Et9VNW9-^NW#4Ey@m>`41SW`E%Ek!qW(@#gvjf3Tez zZo&~XCVpY?H!$!)+hXon0^ucCVk%?)deRJq%12$56l7ZoZ!@dQF+Ao~RKf9i_yzc~ zg)*^cg~atUt30cC_%PbCjWx%zl`}b;bNuj3Jy!|9sY+uJ)!{{^WCZ($d3Tz8tQAM4 z$-Js4m94Bw&msHK37sY&)B5PNUL7(FW_u(tm z1BzO=KC#K|#65-UdNOgMD^sX14ee`1f>tMB_m=YE8eJPteq zJT_SjB)m_-FYS292%fhD!|dqg-wtH)@-Z^f5N5N29HW8H zTj^!{j0}^465CT`g4{L3;lyC=C-m05a@S`zU{*?P!@by5)rM*rcJz2nt$r)wMVJ!Z zBe}6QCH}SI5=G|K#U+?_Okce?XD>%qocOnshb^zmJbk(iJ{!~U=s&erFwEhSql#90 zY(Vthv3uitzzSWbquH_0JLeJ`DFHh~T#TJy8@u|EQ=hc)m zp*ZJYH~U`9sZ(8~x{KWLt|w*0b%E>7|)yq*ly9&gJT8_tdLC zCz>JTqliM0d&LHj*!`mGEK);f?R@1KD%qG09_H?6k)ni5}W`tn2nzlLi(j zY*hA5v^edtqlo&B?$vW6ETp1Zx&{+XxAE}^Tq>dKi(ZCs9g8Cp7CV#}uVv3XUzTHJ zM|F-fREPi|dOR@&{}2imBMbtdt7YJ9O4Li#PMav|hIhDp(f*}pBLF@iax>4Uy<=DW z{*;zK5wpza_|pbcrDWtOl*w*g^|Dh?I? zAh+N@LRTLwj>ZR!Dl?l5qEJI zLqk^uoxVrrQkR1Vp1xbtD`Fysu`H+?(SIA!lx~;M_^7aAR^`l6ITF@fCGjolMLFvu z(jWoow*j*Rv(S~pG3e@4yg8+3oOd3S!e8gKdUgcbH6iL=vKvAZee?$F9ZUXMXrdaD6`L-%VP*C$;jI#r0GT-7=d^c7$D}W8!G9;q zvmA^qm>#k_+lU6k6DHucLjEMt*^G>2{_-wt!_Eb5?L~UYmU+pO$(}gfu(I`Gq~_yd zc6PPsI?EaU6aLY>4EF0!dk~ZVup$*1T**i0A=xGfCkqNU9x<~+JO4CD!dl5k zTi<5<+GYB^YqhnP1s9yXSAOnHD8dIT6$%*8PWpO|L#h^yh^RQCSWTjB(Vfww*s#^d z4lW!5RF-;MTGWCMvrHq+Xhn>mqt3`p>CS5sF zKKTbF4A-EZ;lhg_!j$B5?_1f?C)TW#3~hObIyLBXdj<8ItQpQWE?lY{ZNES43C?dc zzni)~OYy{$QbEJ&^7ZaNdNfXlVE)NrqZF%|YQgL&yBe;)Cfz7GdR0^3f?qB1cBMjJ zJ@=<4qRAT}y(go>8}N-?N0a!#G|TPh`*1MRqNX{UQ1x*?6KdkW$pfuGQNfg zE`hh9iWf_|j-5c-H}_)Q#<2NEbQ=Hy97BNm4nH)db*7hc%~?EG!F!tnc+?xKDe?w;8hBVpF8Rg_*TP%_>pCV{eADLE_scf4?XirT zz$-JH?p`rA){y7Mq1|T30C+50w-XIDxZ6}1Ifx&e(zGVz+Q%&tKrCJ~P5G?;CH~~a zA7oyfnxpo&{_yob4#N8ON-jGs8g1?j^l)nt-()NYRe{>PyC*4%fXWcos5!p?fC-qk zd}kBPc#W?imCfp0TzBaY6wbzreKR(RfFZd&JrQL#Qe7;yOXN1ti$txC+_W(Ma<9g=K}g2tSVUx318-GF6JF+w+*W7 zD>BGZztW~?X5p1x<})XWR&K7Xr4vuLD3Q&1Rc$OSa#D_>J;}Prsmg85$)z=(?W)WW z?lkwlXa0|Y-v2?M@co)y(|$Xrn2aIm@k^W6Oz!YV@QTN|I87 zO!IX_Nf{UZoQmsRjLqVQB8&b3DY0`&O3Bo@ig3!k7U<`-Jz>4=t$*7q_Uv6L6@is* zSrey<^jP*~rl~OIlqjN_g60~hOOp`_VeJE!(NARGPqST_@<@1g2+n+3xe0}a*G&a8 z)3=Yt@D~%vMMXs=QNAdiw4B)Hx>u?*?)sm>i0`leuxCg_IGrn2BT9GR^hOc&C};V_ zSfvveG4wkD(aOLaWp|dIDSF(ObfdLJVflqMSkS;0@mvzl7ntJmRI+E*WONguB$=h1 z?mqINBYVzrXHCGg>J8!h zKDH!9T(rfgR+Qu+>K>P3)o^?*1}0%un{C?TR)lgw{MlQPbmMAnBYvzFkzWEWwK<eb&EEeZvBRWWI?(KrrKh?dk2ik7bB5)yQUX}Z8z6IPl@-CTP~UXM z>)IQBEpcV&)wIi8D{uB1P zaAVs@CdKGB{<<|ads;?i9gicA=@z9CeCM4(6~`;UKo+4;@6$tLJcAN~AtKlI@r)xq zeg>Igi!a6~?Z;YF8GE3>Em98lXzJA=I-*T=z5qGlF=az%#bbYSu?CNZ=)R84-#3W% z`r7ksj_Vj0XRod%%AV70)9|2nAB@A8>0K7-j1%o0HDw5U2I5&4q)+VZ1Au(1-;N~` z%QwmlAoGA!EiQ75Yq;nsy4t(jFUx}Ewj007u@k7tG^k` z_aEK!HIKc>)|MK~J_jYKh4tpW$7zeCg00L{YZA{r^1B{;fc46;$#1eI+QUbG9#Z)V z+c%=9>m1kO;xhQ7y+=p&C&;6hMI>eY{!v~Jm$+Bg?&crR3ghPf6R*mz z|D+>-ceCL`TLbSuMdGM^gBOJb=1=xk!V`q~QPz*EEuXMOch5gF2^gvg^3Pdd%r+tq zpf}WpM)5>}>Eo&4Bbkpc_r`5xcpDtXKgj2L)U&neHC2RfL1DuPf&J&NDoqWD*_y*N zrtG>9sZi^MSGC;<>%t4*S7frEy)z_e=V@PH61${!E#y)~aud-yZ&6l(E+0HVjyr}n zAv~W|QzG5cRD=SbReGy$Jolff6aB{jYHC{F#i}N}AbITJx$+do{z132LHc@?jr87x z^!mW~hPYZn%#N0X|562$)1$CvIG}?nW@od+Xs4PeG@J4!wl-4Jy-Ji7t4Zu!TkDLg zC#ImFc(#%1((ydE2BY9LWU+l2aQc}^xQf7b&JC{5}R@R03yi! z`jqIHj->jka{qKiVrrZ={!|wbHI$Vhmk2}rd`YBMOYU{l)YshpJNJBcN(t~2CYNVi z{n4RUS~)-hdiXuvHhre%R`IiaT>WW=0pg+tKLrvg+HYWV&d5ywPERX4hMC$Pi!#GY zrDGtaD|_D(KO`lV(FL8;ku2!V(ue}G!I`I$dg~U%qLDfo0=sI>;D`z1p-t4 zA~&S31%NW?IcxZ0({Hu+Va4WV)e#oW0G!x|x^3%tJzw20Uu5f158pX{phbpW>}pj7 z_yA}xSrmEmo;RKXd-ChsADKwt0Fip&@=~BS^7!u1ERFkq6`&a}NY_WCUwU2vAgFTl z;aHr@bP+jd%mb8q{XQ=>tG590IIu+T#@0F!4~H!{%&hGUu&Ukba-#wVo~0<$5KkXJa}nMpmKL-)5FD39R$nJ)%(G2-((c^tkzU=|Ra=fwkr?&pV7)g=eYr8~ggbXgteMlgjOWdwgBwc30~WrS zsn*64l>U(YN2Wz2fzl(l7)`Ch&SGL>uFy}K`&uarj-oJ2t^gj9AZ}UmcG_=V^>MCX zu9EyZ(iu07gQ;Be>_epf9qD-R>p!sK(DKy2JO3c4w}kx`VjoJ1Ex3YZ^vnjgq~6Uj z&eSr{1e09cH{%mvCw_G#{bE{HG%r=*nIZ}#5+s<-pt5J`bxs?xLV0YvLuv2Obvi=# zh1PT@ccG*nWFDHjT0;{be^EbYM78^|ffcu-CAv{ROnu6x#c_XxCdWNMLtl6#gW~uIoVT! zN7SR%nAD-0pUk8j{FK=<%eOU&YvTJ$`V(!QI{3pORL4J2*34EJcAat*OM{$Y=P?rw_ zHTW?GJxa&8Ad7-^V8%88f`0~!^w|YK!Pn1+a?PuF1D8)K(H;qPb-S7>r{m0+H#gqe zL2LPZxrQ)L88dJq?qzNOm}H)LK3q9A4tz54x)id;Ty6uI0FVb_3X*fQiS#H$??G!Z z$__Lq#6qM<GHq}utr$Uw%Okt3#xJjel+6wl`+w=q|xY4t@q z_!QH-+%-b`THv+OI;?L#nlj@KkfY>|vRmSl3A7J@fLegcDYLwfLh z5A^HS>%!l+8W3{P6?2CG>zQ9eXwDtE0?H0V5R9e$bxTw78m|t)Pt7-bf!i)?*YXnq zYXBHkAo7XOvAgAOJmS}0MW)}qhKLd=&F(tOnI2!=0EHS90^gMP4h_2udweUg zEYl|%Kqo-~o?@C<8$$N}Cqj;2X9QYCO&b)PbNP@}#pH5WeK&b?mqS0!TaE`YQo=x! zMV?q)6V5SUQz~a!x6VOXpi|K1u2aPkIt5C;Vv0xFDwhwY@%L7GW@Swk$GlsIy_l2*V%D%{uT9P;&%9Un23kz!j`Ug*y%ouoQIpH z><+Tc#-M~lLAKSZ-|T83zlftYRcZ*%ALiZ}4MiLu(*K;NlGh0cFa@=|mCPh8pR%QY zsE7?{8E96Cpd)zL?57c{T5e!YJ$X`&EY+{(C@NfFTGdh7M!Vi%yBRmJ`soe3bFGL| zNI)odbXF2i_w~Kk@Ck~9Q=S8*aO7Ui`7SUuGF))C!1t%|JOCH)>y{XF@^@}*9RjV- zIXR$riXNJ}slm@?VU1d2qOJxFfJvhjQGd9(HyqeWVM#}Lus^N;6Q~*pL-oAM&+%lP zNa@$DjEF`LfQO$O9q)thn>&IVMS%I!zxm{XZvzCxc5>$dUvK#~E|3%;@SHG%l!%0% zrqlRd#JY9Tx+4h6xY1LD*CM<}`$u`lKyAR!o_W%4UwHgJmp~NE70o!U;{cMq@YY@* z_B`Iq@bSYRPVBJ_TF#(l1h9|$cjnpo`XPMfqINb;@E{24bgG*=#CYvyyllgx6gT0S z2Lk|^&3q6lH#(OR>7PKO*LW;EG~(K24}VvTqp_6-O?P%B1o6O7AJo0#)qBw6ruos= zf3-t&8L_zRwBV`JfwG_3nl*TbPsPZJlGRl);A#VrpRVct!f&c?*r(fRS&-fsI^94A zM*#vYxv39kx&f$OZax4akACw!GU)fQWAhnffIRC>;Bo8%fCV*e;8epU5g=s%`0d-U zJ=3y=KaKbUX(POx;Rjp+m~zg|%!48YXi>!hWJwdDZ6-13GzMMKw(p;QI)zoa`ErEm zn$q9CSH{BKRaS3y^(JvZqZRew9&P-Tk)iU4>VsL!@e9pzWOnpcNjB1qrP*T(@s@sb}SF z1)c7)u{=B~6y=Yhi4m)+x!Y~6!1qoHecMk|9e0qBUC2e8s4yEEnEns+f?s;n9VQm$ z|Aiwx#XVN3^)EURYZhQGe&pd-^^!EX68+Zc$&11k&e3j@Z7P(F{bZk)wxh&-T}`); zfk{xF&`U<-KlQ484D}nf+t8^um!{nfZ2L1h%6;z_lv@Ay4vNITbB%~2=Zwe#vyAku zXsKQz>4Jg~t-LLSykFpB=`Jl$TYj9S0J}UdQVV_;|8+IH_dBm2TNVJbHj-5?#<*7~` zC{xbfiDrYnQR#UG1!F#j6|tl4w4KB zvb2eQ!xf&YxI+9i|Le5a^E%yLDHB+aN)!q8+gLMKI?7>xq1s`qS1_0*^U%aiGbswR z%s)Nb8aJ_YdDny3L-ph{8(T7;@w6h6Y0%PiA!A{Z>5PZV?#VKVyg10nuRkuO*RV4( z&4TR%Qax#6s9gF!l`k#|2FF+q#Uk?gtb-IEOSp{BPY1BGgXYBLOP8;QBRI2_9bh6!EJ85d;UjEPE1kA*_EN#+U?GN+DMJc&rodX;RbI(AgrB;IYGbH_iItl{xMUW1OqS+~OR+8Q`Qb#Gk>87F+DMS46# z--+!qmLef#9*3Ea4@sI$AhZU%T1oknA5{nxG=`wm7tT>#@_Z^Bt6oV-=Pu4@IcsU6jth=bn_O_B!@@@8d{RF=!LMCY`0Q8P1VX&=73nR2Sm5-U4fch@MuB=syqjz9g7 z{gEw)kn)kd{&maw^*sr7F9P4_icoZdywG|E8k%GW`ogTUiY=VP!C}{1f}1bKZ(?3K=QYT6 z)9E5lT+>+?ooJYv-qjRD9V}8_(1Q!qJFu6<+iIXt4-Mt0(mhPeo3dKVms+Tao? z%2th=c&wgRf`nwxKVLMHHB~2ADOh%5V1y(*sEe1>8EnIq%IMm4!}Z`^yMH^4H}+N3 zX}Md)a>@qlwd+O0=KS14JHT>a)$r88mt-L+REy4pc`&T7znm;i!bK_ivuTAkje@PV zf0;m<=)H#sawTz+Nu}i^s-VciAC{UCE94(^O(|H*UZf68jfr zRo4_>c`iRW&_hN(n4k0aShI-N3k(i)X=k9Hv$ch^Y!wY`a`RVawx`*)CacF0v&CwP zb+=?H3b=gVo2*7wp44wQp~bc_i$D?9Pn-YI_b_|lwr?`s0;Tg%1 zgn?#4_i9|J6Tp(%^`{KhIhVbEx2k*GzMr}5ass>~f#lfK`)KzTR-s~kW&^02+B-7M zk)%E7b$Y-}7pec!7p4a^v6>k$Ql1$l6D~wYR_0y#-9x1I~6MTh9 zU4J0T#P3JdeMJ!kD7%<;gbQ0k6@k?A4QAX~En!{GU#w!5)vIykwkMV`?CRZBwURdZ zr3$E0muk-Dy4}!C$;yLx+r}OzH!6?#TuPg8eNIOiq+plCeQcuu4M%g&|Ei(>J5Slw zdh3FkvbQ!)f~4z~7*MW^s2mG+T~dvx z$vc^`NjI^!>b{TT5CNXOcessGl$Bd9$DQvX-+_=iXq#De!0TpEWsR$2ynOla)UNY*Aks}g8({6?1X!0Uj3%BL zANNn}G={VQ9tPSZ&r9?(d_2}eG60bgIEYKv6TRufiq{X9gD_1h9vi?Ryz`h73FfFB z{O{}?&5vz2;frQgB<}y#|3~1a9Pbj}cyQ@&;&ae3B_#a!*;u8x9?_ehO3#h%gGYJ( zm<&m8nN)JpP@tc1K~T!bRa5cbm9kbk95}j18pxTa*eS`tpw2&P_}bLtFsO4&SJ1GI#W%FUp+B8ABbd5p zPW~DtwdEl4DB94L>us86=!~dr0T0^PQEc_&O24Rt%|5$&=WP7Mz=sVKx%{>XOACZ*#6?Z!rh2B`chGv=VAhL}Uf8CxGK(e+ab&d?j!g%E@)! zwxoPyASL*Pk>PT3NJ@TSOG-2q?Lu?-#A4<7x{>96y@L}1zUy8ryj3^y`=cn^PBsJE$uIo zIU|TLXfbAUAB$a#@|~(ty0Jt9)n2wDXqMHLSW)idw`P@bPKrvF9LYM!m?6QWumInC zw*@}Grsh@kl!-Qzb?kI==6TM;KVri_uNpeRR>gEdRVP$1%sIayA`~P~ra9gvR5g^A z455#A!>#NuNbW$Qw|2|c$AWVi}xSidV%i5J#^q&&nYzw&;9>*4Y0EFdrXXF4qU3cvWR5>kxy&lbCUKc^$tcJ&ym z$S2x6!gw|8RnPyfR@5LfL~hg`KyCwFQqb6j_@wLkFp$mZpO4+w^wNy4AJpJV$`@!2 zJE&Z8b5~|ft!zhAZKbNGC4$MU zgqHN{FMp^XP^+F5UvmJ+S7LUwV>VthDs`3I%5k+&RN__T>q@t65#i#wo*%)ec2&Gf{u)!PcWIlhy`JY&c9REm z#EEyfUpG|(M8*kee`i)}Xd)8unT_esHq?uoJZL@IJFl3=`&rPX(7la4AQs8fd|~+5 ziU(b+0af`4U@a2_J**ekp9i`**2v1r`B7thW84F+RfF$wX_lQD0Ky6&L~YJZ3`-1n zsEo%%#n^8*=q2=hN8Vq|Z{SU&PxLd^=gqSWbl2aa?pev=mF5YF?NBG{p;^k0bv~-L zQe-QdM@L0?2@Vw{ur$C0)XNCg94e8cr54v}!8@kO0x2uSP$I?lFDHi$#v=s>7qNY= zHs))kTTWjd#f*q*XdHuwk9)cQS)|LI66zKH(fML!z)qB4pC;aD*-(Rr=y?`{qo7qQ z7tA&RdMmcrHUuj$Lm|*Mn+|(1pMP+shOo8QQnmFe;cf15CIYqS*7 zUA?V?hRn!6-aP;5!g%+upQF_8&$OkXsuOR9#(kk(C}C=-6=l%dAoV)A2lt*H8O_h9 z$j`w!JeuD)4QKZ{dHk0;ZZ*7eqDx zFM|IC{UO47z1_;F&>VeUzBZHnUG^zFKzh$S^J8~q-cK{qU z+mZ5?NzPLegFMYt1ynAYHr!zqVeI+gHTj){4DD`USNPu7fI3hx(iI$5rFbTu!6sq1i@eR(8XB)i3SioNaQ$#&y@_w6bDB2J1mW zZ8y3NY}XP4L`349nLQkpM)g)|&}tQZb@xFh3&GSgO0Wy$-B7`t3Cq=B%vu7e;3WH< zrGF7q|M$TfbpM-s(MuWA%1I>E+Au7}FRCO#{OmHwG~k8l{l!wT&4Bnx%e)~g)mRfb zhT5e^f?L%AfN7UVE1@(&#IP!LA#Z%wlq!Hhui~D#pl&@0{Hf;UURlatSf)WK>x7;( zS*b-KY4*hL@`iNZHn&;WeJ!J#DIC_~Lpr#_oVc1_cT~-`T|a?HfJ(h8Y`f8N&f$wW zdoNgRNDxN}XkUarqMpk%a$H}`MszV0++TDxe){6viEf@2cImj^c&1AXRA;!=f%3&~ z=ZuTSLzAmXyIVXr0YCvYW4-}$!T-JRF|jSgD1Poe65B<1PZa|HF2IG;`+opSk946$ z*b45N9ehs-R$eV_uh$?8X!DfoA0>J|U}}KwR2i+uF2+o{v;sdNhl*7>dLMwc(cOPu zkx2*gLqFJl4h@!PginzrC*p>sB-wO)NJ4mtYuJKM=w&l8j;gAiZhulE#Ay_C1-(pD?zrK(x3kFhBr{VHx;WEnl&Tnn-OI9A?(F+$g8q34 zOQiTC`S-DzkK~{EMZfopVrG|@?~+&5WXfgIWB@&&axwomtt8;lEkw)8Osewf%6v#G zZLxw(Ry zE4Zi%UPqc`a9Hb@w>$X16LBGlrK(WW6X+bH;uA+>v+tHa9c+53<;C9X(+uI1zG8U2 z;i}B->h*UOq2qbKGY>y&Gi9RTuUpNUvHXFSX92n2ZK>i_k)s|Hluy?^FnW!d$>7cA zVlBH7)q`s2M%-MTG$PH;EpBla214`%dv3+8qnEZLhwHf=du6v9y9%r6SR3{0>@5k| zljN3&;i3My`$mZh8ghPQRdxE28+LVpUjn~Kc@^^_pK9~i-EnM@iEF$+q#mPzi(aQy z&KR8M-9QaPF-VXGBaT*?QeQ0ER5I1##xiQ;=iy`9H;rmK^Wb35wA}8fv)8)U{yx8M z-QIPgrMGPAPcH-P8Jx!y`E{X%JO>gz@F+JrB0BBgRh{-OuJS4^46t$m4$T}r*(M42 z`7l0&s|Idtm6d4S0EjB_DemyP1hkAu!!-km=mih_kpwHp8Tyxix8sUi+jL|T&l0`0 z4WM&@g9!c;eBagDYqLZT!`$Hg$&EINB1QKBalL_TVFFLXn=jcXkER%^9QH{le)5m@ zzv$9XAjvNM7)k9)>7a zQHWpvpVqrR_}{#ghS(w2Zxy7`^Ol(NSdM-Jo_{Ws z;7>Pnr9$+cB_tJOQF3-jG3BBhGE*Y$z(WHwFW#&49ln^g>yI^{h#2FpUD7+5(y1Rx zqjz-ddBwY{F(LU`sdcq!-1oI50EzDA)^q9Ji+6ZX;sz*_9x zZIxXPyC(q(Z|Cb@N+f(@0o3H=o(X!Ll~@FN)o2!9qDM;h?8w?wiEd7eLuAjiExh@# z;vJ>^6(kg+81>Zi~kI&Hp-w1LVC~TRox&hq-b7Y2iHol`!UR6FU-|==wT9{t}dUzm--NFjQ~1pvyvLgY%EO&!}#t&R2o*qh@{}HyE8C zh@(@mikYvflPT1hI0D)}dWyTxPOwM( zT}bkI^j!r-!+9h4Tlzmt;-rS3btROvTEq)3sICut?4%p-pSCQgY(Yp9)}r+||5*!d z3UAe+6!ic3U9&V%2g4b&21gJzuScUp!bKz{CErL&{`k>S_UOUGmiGhSzxN5Y^7HdO z$<3pmAGc()jHk5xBbJ^?PX1j`P*C8VDZ5>mxmC1$kKu0Q9&k(lvxiTeRoO}oCdSDt zvnfdCvQMsrpnhtB@=+qyiro3^6H9kS1C>ZDS(7E({*ryAR2i2G7A&zEh$D-Q=DoBS zD-I~6I`ilz%lbc_q+*MUO_kiBTn>Rn%E9Q0uxQH?=VQv-%Unm=Y#UW%`_b#C5a2W&3sn{Bb0Aj}W7z*xngW4%B*DYG|kFa!XUrs!CAj^ z_a!#GVxNw2@%!=aOkCBL#O<4m^_}mR1EqY#j}N~OzSrZkg!!cq(hJWEj&sFe1V-HoqmtYjB@87-KK`ZzIJ{dW}c`w^4A(RsZ| z@_?uxTnk2h@+%d@A+dfKikuYQ7*@0I?82dPaD`sQu;n^(Kd(m`E9bffWdh7|(un)Y z2Q(RV{OJnpGH+v&izCgtpMB*4a`QJ4w02*?Hu&kerVj zOI1}hu`gwDs-^YP)8#Q;#2BqdTiszdtfX#Zcrl`8p|%HZt(^=kbeAzRbK?`u&}*#% z!5-HePU7*NoMW+6REbnbYZ%&IltWaueW z2-?DjgCJxtKRWw4w^GC-750a9^?^mcv z6A+WdSB^<-TC^Kmxy5T~pyhQ_E0?Z^eV*Hb&`|j`pIFr)10jd4|LjhsLFV&k^ohI0wLm|v5Y2_uV!kw}Z{6e~z; zYZgrxbj_fvs=dk2mmPchYfYpeM+heE<4v23(3TYd)o25g;Mk@BkFP+u92#x{kK*uH zA4}K9O9|pay91G&<{QJ?aG(SL0qOyGo2b2AV%S@Uu}MGa5|ls;AR+nX^46j)u)3|m zFg;24IxUG9Z`sznMjH3H^i%?+0rt(hwa)d4*dbOAh`GqaZogDnq#Jf2*Z85o*oDasT#bt%^T2k_Pk;WM>z9rhycDTc>M z?R~i~&ctTZGBu~1iWq*lA0-JtJmTVl`rd05Il0SB(+$dnn(mVekv~6tqM)U0Nt-K`Tyv8>!>K#w*7za?N*eOmR3?)x?xL7OUDc#&Co-)TcuOF2Z8|en=j-g}V`QGFEuHWyem%r47Gqcux&2`psd`|8(zU1BGZdP_>nOAr0|{wi+cTB3HGa}qF_YT7$|?sDNg%9oxnn4}-g%Jeufjues~cF0ZwMVO}s4T(BDz%oboutf?vnCw|z-}2nE1Aw} zosXX#F-)|?^KlELs1+o#2dB96#3T)E5Rba`l_G3$&;IN;;`wY>O-Vc{nwY< zFA@6lrS(bm^AUny!+U$n(}o2iv#6o=JHTbYnnP_ZC)!WqnqjXzTUdms&)AgdD2#>q zOCDb5kd94H#HrustpW3G=K!ZpKL%&8&mC|cS0M2>)rX7b_n`o8MiLG5wP+~ED-7@q zBR8=cD%18O-cp=v3zT>9oh2|-M3@BLFB3^(qpy{IRq~PZBk!?!te29L9_+q*I8xFX z5O>nE+Y))v(RCMm5%tCE^LI$~&s}?Q(X{M0uyD_GfH=)4I$RxKs*tWbz*js8Y0-?-3c_C-VyQ@9l!Cy& zo&?N4$=*0SrKJHrjzcWJDVdq1X?dW!J)~`-qhE^uzfCn5{x7uLO%n|?l}OmPT}`$d zW0=FIwleihdt9ZlG^=e$t7Cx0F*66})4Qn%d? z3?pU-Vzt)-X|=$ru!HgRI!gn=ZdFnpU13G(O-a?lm{yS?tLFFV)@O%2C1Ag;G(XcD zhmK+RufW<&c;>gMo<-A)&TZMroAnwWn=Hb>Zq0NY?E)QsZv>E z5743G(r+2waQR~m7U2IIko?Q`p46oukDH1Cs!{SX=Yzl&>}*_M-0QH!8gWu(#)oyP zsa}phG5gWRkk5WfBUcFOx)DWshvu!V4T~Yj} zTpJPN=fC1HnF*`;W}9-aYlzAA)Obe*Pr>wBAwJEHN$=`-o!I)wyD@zZUIzAFW+|&R zYO)>XOukYTk(?=j4R5K-`>PtE>b){u);0gD?Jr-`g!vxO*&Df;qXif^p6E(SX}%?m zvU#!VoDw3!c%T3Br5)la0{-KOyEnNm%#uFBle>t_Ik3QI;i6@?Ws`)r1p$K(-%T_u z4Mi?^KT?f1=eHL|pk~r}Zk!wf1`;F}QL87FVyE-}cQg3^&i?SP+`8R}%^6uN#l;Jh6_E^dextweodfArYqoxU}zg9 z5bN5;tbOWsDPBLglCajmxVBMp(U%L^EM5h2{21i%)i7*kjNzK(*0i!6k`7XKTJ_ei z>aw(XVo^722se6HF03U`D%pAS$Zw79~J=~0{)H~k&!acEeW#e(sz%VN3@;4a&UYTDvy0d6p$ z!G=0r9{{d7(jVu$`-??a*noXp29I6l+xZ_A`0n*x>MxzK?;*16eFoNnC2x9)3Tnh2 zG!#S7-GT@<`=gfJJ|S}}?Ky!uJ6iz6!2G5I=HnjAO(1Lavx6vq7AJ_90P;un z%ak}9N%|LU7JIeDylz*;JTVD%Gic$w;JZ{c>r>wV{5c&u9KmGu=PL;Q*`!96wAp}e zGwOcEcWYlPU!?3yf@#4LMbJW!_UBbH73Fj)s*PF2w~4{yIvPV#>Ox4Neu!wlKxC3J zHg?3gGwd_0+JU#`(V_!_sZapOS@!zCc3`f?f3sKxyoMPm)Gc@zGvDXw20r9^Jxg~zGFe-IBuINR;InT=HooA(esYDHI$Z4 zKR9vf9|b{7ozX&fu^nw$I)SH`I_ul9BOI8BbXBHtl*W*MHQh>&pr6_1i0MNA&$#2J z)AN8$KdiF+&dK?(^!70z2A8#!0h|W>ZYS5`$Qu3ht7sM%gS8wzpt1 z-Qc_kM+)$K2Lj+u`K^OvZU6PM8MefX2|wp&l#Hs;G+GG9O7WdhdV?ssDAOw7H`f^w zQxjbnu2rfUNpX*ZOx=uC)b{bcxYqDjO^C@ASM&rZ)(W&v8J%V{W9`NA%uVGHDgy_d zgLyQ1Q=Obze6IJ3p4mT7BzZk7Om=1dz;j#{VFbI)q;ggV!Ss?WY64VpUmN4KXU>w3 z^j}j<50;O4@6%Q4|K{fN6I}S}Y8t4o{|=ljtHpU8Vt;Rh}dsCBvxB}WIBFqQp0yHs^R z#Y7zs)5X`55&8c%#>6?Xb-yMT;`MUU8iosEib7uSs%^PH6Vq);7WH!C>F$vt!kp8$TqlBfTE zS+eg7GhYIUT5|J8A?^*Me9uT@IIWvrsPOo`+YbNwX)8mSWk|M$za?_hjO`2c+mzy` z?_=2SuCB^NM!WB)Qr8;p&Tc6bcJSN)-;L_qOiPt7FhM~iO6vQ;mz{ON!Z#y%x>kEn zep>HAYm^giL{buvN0v-7d42@&S5wC^<&d$Uh=to+gt3T}DoNs2h{Dc?8~&&bxAm@y zln&kd#Y}-$`|1&)_NBUrZMH1qh)1Cj#fLnZt~ss?%u_izIa8#nbG(wtm5DPA|A5VM2*Y0kLh*1?F`#3l}J-; zEKect(3Nj-E@yn=RA2b^AHNyo%^Agwmg?=Gba*qbH(1YQsHHa{(0zXvPX~8u<-9BI z7ac-^xp9$494uu%)t7GrGW92&yk8w4ysW!pK11`Wzi|cRTYfE7%(IN8mwxm-X2)>} zj4sm7fTd-?S@5Lcg~vrZU-AtFoTi3*hM=5I)RtU>$G)wE&oF;Z9G@CD=&VJjk|oaN zY@D`QrdpgdgO3eoL=8So)n}|i(1l1DQ}`JJBpH=DY_Q6<@X2Ct!GLM_dM_6Y40ys2 z;5R1|1I<_yGzfYyEyD}|4;-eHHWvWv44SdZ+Oc_S%KNkfz^WOnG}Rw9V+F3f74tn6 zSf30r-N}l-KUTQfk-Ze;3=l`IYNpEp^BmSb2v~Rn=%ZygGV4)tf%6@-yE>7b(cJO! z>R!$;b+cwYv)~iEpcMB|hseM{W~*C#Bc_NHkzr8mcxc_{U;%a^o{x)i3KS%5Lll1q zcjzUTupHP+C2s1>@-mpt?lz-D#=Il*_a6zHWk+hYP~6p0%y@LzqlFwpTLN(dkiGxe zmVfY%|B4#K33fJt={#zXs8fy#$+dZpGbd;Fh=ZEwAt~VvC>jn5{TS?A(*>e&4zd%a z8Wi~!-anApEelQ>GeN25U5iv-o$Xc(XLG6qVB&8|95yx6jy*mJT?pXVyPP7mcO@69 zcU#N7KAw3_(|h*7pe~Xp6MN;2@cOk9s3Nj{b;2GLD+;Gd2Yxm!4vg!_1-7Khx=sB> zQ*5wc%=1;HgN6$blN46vWn$elLdSm$c*>+i>9Yfkk9L7aK>1+RYnP|J?ietj`B4-$ zfo?CYKLhsiY3K&5W$qURhl&ahEu|?HRQF8UsuEcq~Lo)WM zN`5?Xc5`2|W!Lz@zfapCaXLHw!$XW z=e0PYd8C8qvQ@Ed%+;CWg8vJ1+xHFr4gy#DRot666% zbJwa`is>G^wg)+do_fBXYW^qae8D#87-Omf4G}8$@F5DTj`e=;jOE_z48P(sy~$ zPc_wjS$*;eTkc}TuS?1X&M4pTO(pGghluoO&E8J7Cf{g_MLVZk`AA=o)7fVwO-{FZ z#aX`R{Xq9ciYGglgS>m-!e;h8o@={lA)N(%URPsiUYBu1uMQi@L;x2oOH%lApABok zq=#m?Yln?u;*PTl&%nb5#3PCn8Di~%hUZC?4$E~dJAG>+ue;Ujf7rQap}pH+YPe3NxI_J7Zl);Zzr#9i78`>mj*Yk(x-DKuPqQPOy4L zgJfsqO>HOpd?=g;3YyfJtF}XO4Nf2j*+$W5d+q4qO^1dB4>G&A9X-CXPlJ6K6HFp| zLX462C(F}Wg*-;W%do9>l(PhS@lZypnkSG8wFew{lW+9tHU|5bP)4lm=4_yD(YsB{ zWiTKPooZ_ng*A&>mlxgl39d=B4Z6D4 zNBOo}Qef%#**}5F;#F~TgFYN&4^3;PRO?9_h)kTSJv=?gWFUrQ+`2Cg2etS8p|3iq(5ovQ2S3BoQt7F@-ur zY&|Po07*UI!Rwg^$k@SKo!+1#VUHd#Z?~AGYhPrnb8sp1X;Eh~yo?(zL?=3^tSdOE zDb;zG_$nmSr(vhjv0yp0$2vU}Bu&Ji2xo`7#L9~x&HURA4? zCnG%zf<8S}F8GhW9TCMsF*wDCx8M82D4CPtAOTa1ha>*5$OCkCxL~=H;>fZ?!BuPk z0;H>ZL|FV10g@jx4Omkqu`q9lR~N3{IWtp6X9&ft6q)+4*B0xnCTxHcO()N#)bq2( z)M?rw6ht3N>{fr~;ONBi#-DVd%aLWzn}joFQ>letcSjEVhCYdj$%9fy#9T>w;46v! zZ)`8l0Af3PNY2>!m2Tp(R_P+@trx#)gfS`NWN-v0^XE$3w;@d;p^{{2(LSiMBjkoV zU?**3rn!Prl0uk}{O1iew0EI}t`*K$!J2WS?Hxa!(`Wd*3*5N^7WlwDGJi zNCQKE+}Tt-ABxlc2N<5x#09`El?6hqd4OATrco+UnI5z>B z8WOkX`&jIkU5kIs>9>zt*OvRy^j*b(#ssQKRF#j%@wnK4_*HvZf_`nl&C|pBCkvrM zU#~QS`75d|H%3i}YshhTxWnrWXw4^zkY%`W(e38wE0r`ZUERvbcWW&D87Q4~e;wsA zyW7gN;wxG`NgE0c(MTQbsC5T3<5dSs<1}PImOu1IR&EoEIbzJi;@yV3@;*&9zFOYO z>L}SggGG$Ay6ogx>1&S>c@doBkUVii)OCe|a=tkuMN%drp-)^netL9asoco;kk~u8 zWq|>Z1+s&*6B1U1gyXh!1u(@hOT~U|{j9*j=@9x@p+IMDX@N`8<2c9vtWW-{Vpv+p zXiBh-ZcIqZAFOFT(L$s6#%5AMPitsdjjLQqv-hW2x;ml?joToE_uv5`io#2v+Kowq z7_&2;Z5$6*7t>8tWcQ#Ltz@UY4Yke5s)4l_N4D;bx7f)XD!ugn!zY&@OVzvhI+IuJ!?j z_rc{kg;Kz;E|~#O)AC}61eQh9G1fDklBHTvQ3-g{yHb=o;;@)sz=_>4>F>;lb*B1} z*N%i5?fe?L3d(~}-W#*udU~%?07tvObd~QopfB+wnxY|hgv5Y+oPJl_zx*&pxv58> zKsKSSpd~}_w9$cZyVvKd*Vg2QCH0u^XA&wJ-a;lV2`Z90bMvwK)Hx?t0m%`9q@Nqn zT@tdK+G~lc(Z0*+CM|C=Q#Y=n2gqr8=)6Z9RwLIEH%$YhOhV*m*xn7|t{;74r~mU^ z0KCH*&)MB{+TA8iyot2LJJ&EYT@NX#$}*&~p!&p~@JaraU?cHk>%2Zy6M{^endUVL zv`=A$l|`MRq_W@A)cF}W-3Aoee_TQkW(+{-`%c=lf!xe|lm5fMl@%D4I!$()F8_!9 zD!dRiSHzBZEC+#B0$3=KOx+|>?}5!+GX8qwr$Xa6nC!qBt)^l`u;23yh-+_Fq#}H8 z!CcI%N9>p6lsFHfL>t5L|LWWhx27h=AvPBfWk zjZC+mspuDF>Ur88zF6wW9cWM3UK03w(BqU``|PCOKFg<2`o@Gq8aH4h$cEI8pu{U_ zt{CA~J|{p6<$YFY9~s2+bkDCGBAld5`4pwUF2$!_bg|Q@0aghN=ryupX6CfYXBJM7 zV0iy=ppwc%l#75GaG?}r?)aOjI%O&HGQ{+Z1b6FYEBg%9Hd1spB)q}grL_3EUPknN z-Zz=MX}Q5RPXI!+4E;lwF9C58J6pv-gZG{?+wI(w=n^1}oSm3r3y-0Y^o#3CEPI4_ zujj+fp|R~R&wrCkKvNXwj{C@KU<{@NZ^o;PtH(BG@VFZ5x26#WF8~+Knp-A%eosKSpYyD*S4}SLZ^)*>Q|&C1_*aWcq(b55@Gp7A zk2n}#HwT!m6c_*{Tyt(Y`)A< z>f67~)*f_Ra`7^6(z)fl(^54jXUVHe>hQ7a3w>fy3uc>zK-;0)`=aHHFU*(H6wME` zcNSHgcx2e)Sl7&dtdOZO+nVHw>xGYId0ihBrc6L|!}6f3%RSYqEnX9A0}zeqA7fnf z=f+eaQXX4qmY$ZtD!*xJ>-i`j_3(Pq2xKea>LQz9E}D~&gFS}RaBMvE^MVJ$o!dX( zw;GzGhp-uy7?^KIr;I0Wm4)U zf;9+EJhO-4Fj8_#d<8MlK55n#&KRMhGW(qzlk@XSPyas?q@rStH@cT)fE!c`z;!wt zu60xhNK^b$n^SoPI?Xv_>*?_k0|4iFPal$MUA+PenZr^uCdz$N@gOTrTG#~%06ajc z+>{CHvbfAXG&*KP9u;l=TBZO{jPk#c%5=OojU!^$+807jNRzMXDW*FQ&1)+3*%i;u zmbw`xyfqA;AC@yeWM%J7d`#Uqc(cd-XWaYBTy}LPhp#Q5jYjF>x+x~g zm!}3ff-Xg#NbkW0s!vRVw}R&mUuua>VJVz0=eObj6e<0le>})&)3hA`t^pShp4Wcu zh;JbDe;fe#Ez3?)n?N$egl@|^ZHdN$V;ipM21*Bb#IeHHT`%-RJ+YUa9QI)_nB2Fv z82xJfYC5M5cr{W1(9_k?Qw(SO+${|sV??_eH^N3Q_JswD5+|G;*XC_|PHd*eCi-Pa z2!vg;H!Rcv<}7_1nqr&Q;Z%HX~LdP&AA@(aL{ZH@~X7e<0m2$g#@N9YVID zuhGY&&+fu(%d$l8X63ktr__Nk(6&5el!5Ne>QkpC5+@M_2%$envfDKl-7dSE^D~s{dN)lFFh6+rh`C~#sKKa%1sp|B+Kyg50a|_`D8H->Pg}9 zJ-c2j_f(#|?FMg6>73T&B6iQs*_Wak@f;DpDI+k`6j0J8Rg3e3-_?h~E>@N!>{6;@ zSBcLpPbDdgR71Nb-hx#fb7YN|bD=XG|M=}C1N;e+LP}n9n;^zoW4|eQ^bv+sX#O9+ z{gJtLpvZPNmW`FShvXt*Y%IueOpTkJxO;P?_qh#o_%92MGp_hMP_sBXSgH z1N9OwSsXGUw$hn;dLgeRii;yK?x$i0%G&Cd zTQxfq{VV}j5y5j%fglcZzSwY62hS}EBn)BMR`^)^W1$9z;8?XXbHci^{hL)=v5Cwg ztp@d`y2bf`gPe{jjEL9qn)wBTk(r4gdL&K{XupJP4WD9hSjM&v*9!o>uK{h{qkge} zX(KZqMGx~F7y@imU8yxYJR{N-pKamD7RE z_1NU>Oa|hO*#dx{K&-r6u031DiiYEKZZ^I}RZrCe^wC(2tqPdbucOKZ?6uldaGLx5 z-d;+ApVM!I6U~U#bZ=47^&a&^#K&*V$Q#8a0Gn6BFQMZyE_7U2`O(He;$;fTnMX6c zL0v}9;!nkwKRrk?Y}kH>8jDNk<>fwqx@J@N)$Y;^&|nhek$iMdOIICIomfMx%hSq6 z2ydJYJ?*QR*fl8pzGgORZp~Y~nltIJZnu5n#6DE5pQ{qUN*qY&4Mjy>^pqHI*c>*9 z`FaU+Z{VvTqCUXH{+^dr5*?)a>HKoYdcRv&0O##k#4|Hv;WTy%rdMkvI!YuE} zz{}jGW;aC}K{7v@ml<0iN@3r4^Il7-_QcM3?8U_dsx0&gqBw`kl%;W~@54k;y(xVj z_Nh@$E<>X08U_DW3fSDH?!~tfx#06e&Pn{vzL_z1^mwPRk0e5uS;A#)@(HKPYF4hz zqmNF#8LRR=DjO0c`Pwic9`Fh^eV7|b=MSk@z7*`%GuJ`BLU80AuI4;1`K{+USX~-V zy;fUnilOOP3vggq8d`re z3G(QLie`OH)k3$Zs?st6)DIMK-aKd7kaVoTn3l$6GR* z_2-uty?s!=cml!9A}giK71v2Vn%Jg8MXyn=P1Y3CVfd;YDx!+RP z@YBu7CgoMa>3O-;?y+aE6G}siG9x-|#n*C;r$C-R6L;jyRz-)Qn!~=qlQ(>RDo~3=NFy0 zY-AS0x_b%}fb^gMTd{3>_YNXY0W2yY{YTNCO!M!a69cg1Yl{P9+477xma-=$e$-aC zQDGf+^Zq04jhJM7?<@%Kx$sy|zwMVf0qPwula5F)&&PJ@?}kYS zi~MFHV%ezW9m|cKzHxr>+==4&>e2H?qP1z@RRU&bEHz-NZz#ZUAFRGNU6kS+WtH8d z)P&{)I_6Tlm&EU>!2uBa$ujECw0zV5{AeG{L`H^w3=NNt4!0m7Bo4F4TV2g|rl+Q$ zrbCKjF2>3g=pHCpa_OUBdrN;w}4sEAd2giT>Mx$)9&#&H}vm?B6h})@~M%Q;A z=}%du_++L*@*lr3Y{sC3+o}2aYBP!QiM~8t5s4I-QHlZ2Y!s25@(718ttn2_GIovL z{aF^dw0fF-&LuHbRHl=fd`Kx0vW`A(SW4IK5HqhX(Je=cwLncZ8w2i%lFUnJpwrLl zK6mj4NJ-MQIllcMY1Ke@MzdZw~ z>rK$_CVdj}?T4WwGm}QJ7jrYF>;zSYuCDC_5T*Si#F1Hr#vssp>r_U4)$p6l2~hof zXN5W2m*`O!#*rngS9f=?c?SVdoPiOvDNQ{k#B*B_psfKrYT|-Y7FUXhE`Fj8&*!(W z7Xq4B+M_PW2TLXyZvudCap-t7dbwwlM+Shusdkh!Nw-hD|8ln617w~E09WY_px)xq zRW>I6n0a6Dum9Eus=xx|lqzgm-MN)+hw;2gI5(Ut^nmNc$SWSQT)*!<_#SltbP|@z z@3HgkoyB>Wts5FZRjpb1ZJbhB?=1pK{;T0PwIk(oJF{PZapQF)tC7wA6>Vn3fCfh# z((??80(Lb>_W#(^sI76DTEKeGojTLG@njQ#Dnn5b^vSsps0D>GKCh$BQusz}3?LY~ zdG-EGyDVHMSCR&2MhS9sis(vd1INptBuS8RCm?B<9s)GWr5z+olh;j2jH7mbdS-EskjxgdwdJ!87o>0>9CgrlE_(;+>_D1$zjU z`ww|5TjjMAjEzx53Mrk2E}D2Q2QC#O%~~Gi|1h@QM$|JBP^P?8jq#(N%UIJGysedE z`R8xMAHSK)jG3=Tgnh9YkJT#6h9t>pAtlGiX|zTY47KekGD>XFuf}He`TBppFl2qP zejd=ZxnfVkq@*P3nKsTwD9+vOLNIhzHT`jPq1{Y6@_I0bRv)F_YFGwEJGT^TWG><; zOg@pkeL$m~cztF-27_zyC`)OhOYdsxghh82Y#s3aE+*2HYTXBrHCHAJ?seAFJ%0f= zN>g}6`Q63EQ!JFd39~or*n{mV(%iOS4P;fWXf!8%6L77BOn?P1jJx%Lrv_v!CngOW zE^Ow&!?teo&LIYl$%b4!%H=UkGrRE^MX`A*vhpgnGN7`M&YYx>kZO>>H2Rrux`LOw?K&nR|1L7`PEexMQ@D<%CZp!P36US7=Rx$&#K#ldITN`5M&c zEsrW4ulAAVqIKo6_u2`}y%(Q99(^@ZmueUu76|7q zO!I{&e6CH0EAsKO zS{DzjZIYNAXNnwK2+XH#%$#4=%%3Y?41CkESiKgsa5Az(p(3h{?2Y5x)t50k-6BUZCtjNv|%Z$XjDQH5MbLIP=*}=K#kD zKpF`S-)!w3G69xmVC~HLGSBKXr&rvWVN2?OH~V1!>0QKSvy>$<@^w$=1H;p*9b(35 z&o^fDK6J8QlwNUl1 ze@@XDl${7_ZSWimgU_W!EYIIFcSH*p%*bxK?~Iphb#}{)ynMnO3&IcEbiavo+$E@g z^H>-b8DY}*Sb{34lmd@D&4MZ9`}g7II+h(53)Pqwc}G)Tq7S)T8XPKQmdG~Q4VMUR z^9i@Qlgl;KOD-q5PxaF7&4jb0E1Iv9Y4E6y?N2FxQu*eGrmgNTfynm7RF-xAycJx) zi%gX(dT$R)YaHhl@vz$rU*Y$t63x^bEJMw8WIRD%jhL}s9Ej&jjRk1fEin-J84_wb z@%SRUR?EPsiK)Pm5%0Ds3C=8dOv>34Cz_YBxEdoO5wu`B^Iy^SXVq-K%# zRCmrlW!e{eG%z(-p#Rxoyvu3eJOJ;%x>g05eE^n|1iZQFxpg&dy>IRK^&1S#??a9* zQneP!DW%!q;iz3Rp_}kiU^>lnw?v11YoJ93e2nN{Fd-@^GRhaq)ZYaK9Mn-Y%}oM7 zu*(D_aw^^%m^+vQN?`i1K%Z;u!~tc#v$?=u*I*w6bVz^^T?DHyc}44zKX4AC&#Rz| ztxLZ+idR8XGcHM_kR?quU0X6?_XN= zRlU3A_~q2ZU^f_*^3`BYgm&v%Y*;DPcE)*g@P)q*zi}bX02S163R&0<%OCG|&k9lv zKa9pdILyOP@hB_BrSCdKF*? zV_P(@nzA~$&#^x!)Xt^v%9R6?YO40dvB=p%0vF2M_X6tAmwqr4Q_2M z@~l`RoiXy@NGCNt!S!9@uRHQLFxWR4t}Nxv&zf+xu5W5^G!OrtJ(J3h8m5WBq~eKO zeWC5X?pkIz1yXRl5pseE^n~b?Y3SO+Idq}`WO!jw;ouK_+N!MdNhQ|MNq0YrhY?BR zzw>dfQIFfs4RdSCbBHI17BJc>Xjmq}V){D>*DSOiK2ywhVN04;jh)Xzl#oKG;#>!6)+C zy_KtxYLU(vTd^y!k}`kJioCp_(jsFq`G{8i#!b7L=yZCa3c>7`D(27T`qeD9eLSJZWp7PG9bc zct1ep6Sdr@`2;|E;r^4c8`f| zAz`~rP$O2I!U?;3bAGX9`2*W*I~b6$@mlr6cIK#MRQAKFhuNI+Yc-zjF9#=VS#m5W zeSMZH!6&Me2d{;T_slhB)aE3zezK8M>7()Ee?DXm~8 z$&3Viu*>|j(i1xK)!eT9Ub6~{8cZ2=ssqo~;9b1_M*C{E1BXR5+5DbD=j{oVQD-Jz z#&G`BUJY(&$yde~X(3@zMfNwL{2aD?o^=_J-rhojx_Ot3IXg)&;a2k~m;fN0EGWOG zWCZsa@mPWDCAn!SsGk|*PW&tzz6Nc%fVOOx+g~vMQ~1IdpkkNmAM~w#aCda=Nxw<6 zn!uC?=d$31`PFsaH0LH^xVhg9n0n@Ujx(}!b&K15N|j4*yp7;7=N<2N%d&BgG6XYD z*J%gg8*m1NAX~z6(l>p;CHQF3)RMU_r^IzrR24Mzb^q&rhgunS8$JaKLhd2hXY;cJ ze(qBC?_#w5d~-Ae1A8uEQYz0Jkfsj6N;k%1Zhe+Z)pnCv+NGQ)LED#>GmtG{j{y1 zm%7GM7frp}YP^X7AHi}@*n?A7VEq^7SYU4#_(&T_`ljkOQ!~TnD@-C#)JzRxE+7ui zux*g8MGz853NdEsoe&z(tP4z97V)O5}Cm#4llvgwkGDt?%RBIsi{mrj$cbX6Ap2JT+p&q zkrELjgs5#v^s1jI9*4&}R^=d%Pe!>tJSKs>5R;b{}~0Rx958$wnz_~ncc zjf`wFZ`F$41cdhyTz4|l#$r>y>;S>A&8{T$xsanH%l|D^6 zgJU-t5l+zc0B0%t-Cc1;TB^B!u3R2;hjCJ}A$j}ds_tAqLJ0;~X5fq}ds&Z5^AqNk z)GPv0NSIc8-vCL=r%o9H)u~tIV(ghL>_%i<2JIvvGKAj?3r4~B<*}Q`-6hRjO>yYn<+cPza-Ti4*@*ny`_yKVtGAfXf0@2LW<6Q!DyNJ_nx7Hj>L18>GO z9daarwkA7nV`fIJ+7L52w-|q9k|y)$0gI|CG@&nvskC0+UQ-IklD`Uu$Td!3jOHU_ zhRZ^=qd?U5JLib>j1sLP+0w828bao5RBTAW{qb$vu3L@Ula*5n%R}juI8e%c>)MUd z4etEG?#g8R#W8TmB6gO`Peyve1hK)(+nLwkR}-doY^j17xccl`7Gc0y1p2nM2Zs&G zSI5}2`}hpmtSUV?Smr;^haLZzi4l4&-~O?XE2;?^S<4D(FEub*#OwY;==4P{JN{o8 zDxehIqlClI`8-ZhDyzwJIIhpn7kNg`-RR}d8g6pb4!-!yJ+$RO9oy}zmKao4)L{E zDLVIW(yRp(U9g61RC@faf0@JYXekj$dHT@>=mpT(lC~U2y{{cGZNe5h^y%bf4mTA& zp$6E$G|a1^tR2i$UHBWYfrUuz65`?#f?ddw&HP?QFD_(66;S&N8k=71F-bS~5`~Ah zMu)NPNr><7QP!of_TJFi%sIbV5PE_qn94)%@cNya;3SlP{T0e!MD+^wDz$xhN!lry zD>V63okJwSQ#iwxh%)LzI101(Bk;UV#p^0#S~Zo=YP(#&vflZyxb?lB{bty$Q_BEc0>wWwO= zqumw_G6z=D>IqHU8UXHXBecojj{1xfT^ zTcLZ-%znOgdsM)Crr0`cdL07oFQD&m-M^8Oz$k%@Kw#eNOU$pFbLZgPD=r>wokJiv z(7w2aeFxvuUzz7E#@`!gEjk118Z*o^s*eW4Zfmb#DGR=!=H{9AU&?~U$Cqb-Ptne} z+!O~~{}^6p{e~l6L*Tt)^P~@}f81GH0M%0p(_pQ1n* z>tv<5dVgR-?$?4Alj#Fq#Pu!Enmg&aEHXwQa_x;C+A3;pk-<$=G*~7H1a!tv9~a4A zyp~E<5TfJE>}UPVQ$@v-Hn2t|S0*_&R?E$7Yt9IA2 za1}i!{iG~=1w@9E;8ZQ}Z;JPF(qMaU7kdkX8rbM*x|BejIwyz!Ppy7uLRH2&!kpyl zwc&}RW*QTGvPaMR;X@vY4V=tLlA+VD{(dbNz)EfQ8UhQQ=;M$KGR>+^mrhk~WgDQt zPz9hGXKN`jHwdL(P6{1>A)@Zw!@1@h~Kr~h=HJb1E{bjdp4VQr%6 z7G*9Zjo9hGH|PA~mGF1+>$-4dA=kHM-`^T$J{@GDw4kr~7$PP=#xgFaAQ<{VKzo&y zL!jyM)tEGOQmsEZFHA>67vyhwkbBPC8hObB)?yzTt$wIdJmNZfy)YM}TMs|5JvYS* zD3)~Nt2ME*@j7*yqPW(DWz1I&L=Z1aH-WKxH>jM)Ojj5+4y*GG6Y%uIGp$ZvjgL z9Nh#NNOS4I^e>m^-txarJGoQkQ%#rK-1NS)mwjWrl)rjF2O@ZTtA7Rle|iH`D|;Ge zeVdb*HBS@P42*5) zl@_hxlMoHz_YgYxt6_0#9MR10Wqf@2o!|BJ&Kx%#a8HzxVmQ>_+1Ult$w%_j7%WdR zJl4zf9p3Y#JDO-!2W*$P588Td3S^n=^t8mu2N`%z`2<&a9q&n2^e{5fpV)r(1J%ay zv`gM8%3<*yBEG0L9R_A#Sz)u_0neh^`E!}}3+TxE#wBmy;s?4_QwC^2^LPcz*h_bj zNS^^>YVSe1r{`86)`QnR9Wz+7cZ2;#Z+$sQ>^<-M$8Q1~r1NG(l_}^e|H#)T+34S}lZWC`*zsnqZHpt}*;6F%6KU>>aae z%SUt$gS_7Q4R?2(0qcONb!-gfMYR9UHD?q8B!1M%nT3x}R*p+(#Pum8Rn_`Om?tZC%97`&j#k)miUV4vxeR;X%M^U2C0J zvgYs1BOuE4t^OrW!Q#8D?_P;U4^ZOMlbe<txqo-4l81X0l20v@ zfXmPi-GK^$GFq~rNl}xphm5R(mk2E$DiG-;@RytRH=<`Ad)RE@T5Vu8iU7q zTciSzLRR8_0oK_tXKJdbEtbCHSsns&JBQ3^F=#)yBHd9n!mSCVZZiuHm?uM%BbBTy zyPjd1ozSfcR&BrE^zM6~*7w#Enb4a=f##&d%iQ=I z==Uq$6L@2%8sd07`P_5MwUM)F*tEL`ktH4l1@XSadd@Y$mlu@y)q5+QKnU4-y>m&& zW4a`at^1jFFE(bUKV!SnwGtqyJNPUxw&;Ro#@=LOxU&cMm0V>8D2I)_lFRi6Pk6oq z83bhn!$?z53f2W|cXm2?Bp?9f|D@Aq`hhpgTG&AbAd>~iLWZRZj+o_NbcXEKTi0L2 z$6f!U`er{M1ckwy-P=sjz)MI;zn2~bDhkfPK!Jf~;QrvI2!l2HxbWlxFSYG;nu0;u zW3LLq!6wZAm^b#)U3Y&pJA##f{<$gzChF^e?=80d$4Rx7?m4d^{Q#(4DhU7a8(UYk5n=Py0o4R?86 z(nyu60yek)l$2uj?W=U%3s_E{B>)St<&VfBjqdtOjb*fQ+Y@ki>W6%Zj$M}~hv-zO z=sGB68w{I(-&U7gZJXL?eZ&d2vXW{aYP(>}nX7#OAxLrM|y96XO za$Q<&4%0ecp=VZM9qi$=hNeE|@Ep|9T zfc#Zr|yi>?Be6m9@T1^oC% zHP~-+e2VcK;T7Hodfru#_zM8@Isx<%oExqsu^`vFyeem|a6pLK3I*C>lZx z7%2sx%^wsHktI5<2x~TwY-u_GErl;f*9zM59&)wlXI$SoD4#0Ll@A4S{Se(C4FCU{ zQ8=e;Iz?R z^4Mxd{mu$zu4n49Q2vs~x_fH6KLM#!AU070);LKF>{7ohNk$Kyq1|{C7cn#-6QT~U z12;OYVM@a08tMHbgtr_>Uov^zX+~7Sr6IPEpSw%hJKU|qCLPPu zOOL$a(!UY~7I}g?nw?1j{g}p8ipDxZ8@2Syo17lJ&b_A_=qCK;#=&iyJm_W@6ap#`|y; z?*ib03W0@HaZ*I}DCY?zz%{-~Ib5hgUb&0`o7*r|m*qneZ!DJjo-K#nzJB-r==$og zw(@Q5=~O6ff#MFu-QB_zZGsgq5THnKcV}9PTXBct1T7XES{#DAr?|UIIp0pty)$#p zeZKw$ZJIpK&fe>{*1KNuRl$cC3sMZl2*k?^xW6iR86{W`?h*lg5TJpViKzA)dOZUX z4u3x5L$bw|=5-8klPHd3h+_+~zViwjbS#%^AoChumubcp!cSaTj%-)(Ho)G^N&sn0 z71WPZh-b{Z8<{g_4)AM@l33&^(3w5G3cxE8${8mX=zxL~IyzYM^31SfNi{UstU~Nq z^+{|Is$;#$?t@rm)|So4EYte*V4lo}xMds;Nd!MGz+Pl4mgVl*Cr}hl#`;nB$ixy{ zK8jb*WLG7U|2pYFb(N~}AH^!74RZP3xQKaAc>QU|HJ7NR^qqFV)z2dmpA{C&n(wGa zLw!Y2{_HBiSvJ~Dmhk0r*=C$zVvUv+Ia`3!!ev02`m0XRg*#lKU!g5*TfrxB=mMIr z-l(k5u2z^aqJPHmEb)@-Tilkt7fReCCUBmzdK$IDIhHx;JpL`LC*y=~u>%-PmfzK* z#D1C}j_*-5S2I=noMZa~=wJ_3AQ^mRe0wqF3O(g|nRmG9M916{S9JKb+5p9h9T{zK zurv@1IEh6YR3T0dYtI+agbv@r5A`jYKc4U{D+`YtGNg7qb$s@@)5=z%l6P$`D(jEo zFUvWegN=dTyTu-liYG9D^%T?;BO-cNO}@rgzaWHS`PW#h$1{x5kKc-bNyIgw*u05? z(Q`G$a@GYLpzqN~{sh|;j_v7!-MUXi_*mHpnXL?8vRI5ckxn+zm6(n=H#iw%<6b4Z z$njy3C8DIDI)Wz};fstsOMK7u9)`mMI@z@IMF0~5&fyZR+;>Ji{FU{(9*x*%R^K5K zc-n>upl;mI{~(SSPS&jimzHhE3MBF4lAXc`7Wtl`dvwwLxRTcEs!v zMU1)SZe*q@&Q@uYO2~Et_wCie=aI|VqTMJh3bo!qR4_>l_=Y1{jpLvkbze_1htfQ> z*5vc!-}`><>A=1pYgu0_YiSbg$_NYuNLIlC0l{dJRZzAO4UMjCh=qk2xq_l}$Jep9 zcICBd#p;3Ao6lR6O6qle&F*);Tek~GCOoyH!?a{Hs;GU(Y?XI zO72ogG5BQ9dFRMv*uA(foiWLrd~#2V@^a817n=Tdcka5hevW%XYdM?R&2QO)a93V* z-6w3$a>>&k=zYRv3`Be%FZ|ktCm|~~0p}c`*i~%O?Vba*6u|4@mQ7M(JMKdlmB5g` zYTSyDVAn$G9kih*2|yV)JQ;Iv3G<^vx7jbMFL2%5J^%)66%;MgZMNt>oU{6e)-Hz& zAS(^08@V#?@$)3r`xE`Vy$g4=8=HfMD-)rsw3q#Y&gOKRy9IiZY4Wz+ECQN;hQrmE zV>H|UmRHm%rCOaSvwkk9nhWY7y1GZ%67d>?Ioj8wdB@(oS^P&5_wBeA(9D&k*?3PT)`%oH?L(s+5g zdlpr}_dop?l`ibm4*fN>vw1a(In$x_Y)F0VIZr-dpVWCaUcn^+oZm>QbPnsJGK2%|?Sh$LO4wbfTyO*kqfDRc#Yd-$vwC#|J*vIZNfrQlT9n1nd7O z))gQYB1jUq_j7gaaVeZQm!>T-#S<+~N<$Vuvnq%~vXZ&^GKCu(v1IfXzExDr$M=N# z@Gb;JCIK6Nj^>U*_#4Okd@vs!9VsGTWGo%7p)5c3`23kDz33f3mJ&-?BEgv9 zF#j38CEsBVC)Qdj)R-e^K`OQ;UzY1ZL|M|RrODTWNw`Kt62!fF!<(Ab6l#Z;D2bu{ zSl=M>op5Wd^;ua-#>2%Lt;bVMysNDod$%E9w~TAP-`wPV*(`yvDH3$kW&!?@og44$ zaD~@${2vA7ep%pc`do=saUiX`4(xIhpCn3R-#cri#;$AYQHy$IiSk}ZV(4*oYkt=@ zl*kt%9yYH9SO*EzistRJzQ#KPx1Tzt=e&I8>)c?@bU=6-lCYZ1Jy8e6xhNbB z9*_GXKLEt~LcfFI*q1sGa&m17i0cceOwj7TiWKX(h@J?FP^%>GIg}*VZO}Nut z*+eVHcPH`OMP55|dRJW6dQNsif4zrh4+xUk3(VeP~LhF5* zAJB^eAUo2n#m@u4C5U8G4$n8U@FXw*a_O#lqSi<)ZSPg|8CB*&M4cOOMSXN%J|6|o zERtAO6EWIGYe2kMt1bFdcDQhIsT<0vpE?Y|2)UAFU+2HwE2rKm#Eap!qi;CKo?3ew zt|GLKSG+GYAZ%mGdnK7M7P-EzU~Xz_&yt88`lFtmy|POGr#5V@XFcH-xAb*KhUH)V z8xA!^VUwqtc!n5_G0{legu+Ixm)azGz?qGWu2B@COE-*)YsX35xT#zL#(m_-XPG3Y zPouyvO3|?`^2rYYPS6DGz zx3~`i>C|3i)!0&tBm>LE)5a46&w;B`=IJ^ zPI@2v1C$T?gsR-~u0oa9^_G1r2eN8`Q1p`>_Djm@9RSB#hVFU4YZh|#cAVG zZ1&UG7m3$!9aW75)A(@4_RL zRZE^DKA~CPhD_%ei7a!1k?*Vd=5Z#5(dz3{*ILcD-*K~qvw{=@SH*}0=;eo&E9Y}p zS%Jpksd_Z~e0menIcLl9%=56!3R%`9EKkNCvAb7v0 z#CKz2xj64B$4Ew0uY}rioEE$oI5y*~@-n1YncsBI24N=-$|>bzf-Kf~bXBrexjKxC z^R99(!>hEU>hJQ5XQit z)Ef~2pOU3I|LAG&kbf=9kD&dsN@B%G#1Q;Eb(QDkA%iWjDRSd4iU54H)5048I4N&3 zv&N698>B>~vfwAKWMh*s(=&ZvdE*`$pO(#G}j+{tH#<#ab9nw$|ymf zPdO~!QxZ5_T}s)MMrgpJNA$D)cG}}QS_n6GUAXB|jw~W=bt#|Ar}SOYL%q3m@ak52avgF6d&(GAX>xTpC!5eeMR$H5R<++gY$@lKE zLk<^Fyabgmx0VzT5{KVJn$14K1P+d3;ZnY0R~JAsoSZ$P`ONBC?!wB>=h2y6?NVKS4COmhEwlF_Uae9Y=hT8(x?=LR7jtUoEBvs)3X@$f{8Da{$72_=wgzdsij$cdVN_=EN4Bzo(pv%Q#wR(}4Xk7P=|`=w^CFZUj<+XNa~ zzL5j{#mth;dZ2tNKu^Xw<~$Nk00qx-<|p&B+cN;_+vaQ{JvZ;H`UPd^*DaXBPMb%w=sowcW z4LihKcWLEna)kM)ZKZ12FS6~&*jqgD?ilgpWHgk>s{s5SSECi(DhN+xrngF-|C1Q? ztu8;NvdQHMX!*%pd`(ks%xY)cLGp@qgwTt5c4IqA1szQmflysh;oz0xW(lxV!L|+! z|4PIJ^epdC3CZVf_)w3j==KK3ABWbyRUbFAt%N0V%&8oMTtTp)seg1y1sOJRvs$vF z80tZ_&v9d__t?1iC#?DG;fj~tviFWxVsK{-4Iq7pB&Ih1y%N?Oj;0pyd%2sJBUU?m z&jCUP;IT1ZR`K9|55#~yC7HDt?T1E@dD>4VC0F;n_c&mJ$_W;-FlL|Mh zIp384N39^yFiZCis+W0R6yZTJK><(QVIjJws7C`G9fjj z7MJ{LRM_h;{k=Q_ z0CE5SQbj#H>0S`N=teeC+p^R=kLDjRBYeN;s__PM^X0gsrZ=x}kuIyQwO(lqtF^Z5 zYiY**O|nc{Fj=WQA6JLCnP?(;m@A$y@$^OmF}Y#G*95lXZmjY&tJ4{(qU7X|ndIc& z-e5qWgK}eJWo4Oc!zaXl38-`wr4LdwJTEg;;nu3D@wp8tDH(@^w&6zVO?4HPd+yX3 z68h)~SwigQ(Aw7s05tTJK`Nk8;3~@cv^0y^0?$HMqGx7{pUbbNeps_9_|`#Zvtnvk zVs*{OM%KB~6E*9djJ~I@F9??NH6v!SgWpL@ed9FOHIB50ET1!Rftl|~Rkir)$^4Yt z2btB1{9(Ab`T@~M+&;L2p-HcN&# z%cN>Dq=ejfcTOciey*pw)#qT?xK=xFHm=8Hk<9MON@8ct>0-D}Z9{w2%-1`d>-WV| z?7F;-l-=e_+>G!#-eDjQ0j={Zi8`n#`xO*(hO67PF^%Sq3zVL**;m@FPG|jzUMG=p zbfpmK4I^fEPtgG6v!#KvgI$1Ba}_hsUJez#6-IBrE8Prk>*lTvCDu1f0G+hfg3fv( ztk55wu0@r)a9XFeYvH-Q*@Whe*Yt1h4V+G^13kuZrpC2c1C?O+`m1z4?)EuZ0>!*@156h7%+ErT3#qay|Bj0Yy_qG75#@ADg1!#I?#= zPy1uRNo7H&jWz;pu^q(=r_uoGwEi=bmX6~g-Q39@rh&#eq42Nrul&BJJ-<`X_Lcn^ zqUez5f@|pNPWppkydrd?%vt-W|H+?QOL@%YA8Ev|EHcAHopCNu5kyi2gJ(@bgB2>i z-Ud?E8 z*5OlVqW2z_790hoss|;v=vxh^pH#CaaSUeL!1y<9)pQFQ>07XOlRJc1By&A$K|L?y zn$>34uMv~=nCw=x=7L@sh5f`N7MOb00`==Ee;|zua5Q66Y(zs)sl9zmCQ1$X_>UiA z2eJqrNEdp#Sxiw=+6eOOX#1{Z@n-;~)lzY5TFvVd6ZI|=8-jt>{N@<6L9A$lh~ zA8q~){q&Y~Rdb^#A1J0rP9KGw^Dg@KIo|9_~wei+*W%eT$bj z$N^yJ=y3p1AnxWqKLgf$R2mV^JOn?}!G5gLYLE;+LvAC^SE!GnuC0#F5&evkkEMCPYkphN+|mpPU!Sflci9_XvBYzD z*Es9PnJI+5uNxnq#3T)D1W$Q95K41ef6hNQr+77|_rgGR^r}G)xnuDT!lG1xIQPo(<7umD779v(FqEO1NohPjk#ss0t19|c1@(;*W z!J&*6o3#?3h47!OfWZQSHzAQsCd{OWCnA~JA;|SJE545#wql?{^9kkt<5*-hE6LS< zz80^~wt(wYd1JR!f)cEH7K(6ZiVRADFui=~tfSG5HPMF2oj5=!yjmr+tY~sH-keaFe`Ruf9RQ^~uCi$V_7i;gUrMis{8S0p z;+bSlF1<$|f9Fq4vn7qxa225bJt=m)l3$KWh&a@_H}6ZLa`OO#-mAh?9XusTD}{7e z3|q;_o#gXW(S7$Cc-=JTxo3J4wmaDR$m22C7S7+8*amq$xKLq?XgUX#lRlUEVFTr@ z)N@-!WS9cm$3)Isy>E4fV!9h>>I|3Au+rk?DF9xzgAPGLbzcE4{pL_JKrKD4+}&>t zZtnH}(kyh0LKOhbV8{66Su{|+v535Q(l~{p?sA>>G&7KM5qW~|zHBDvUA8AXZpoMQ z?I?$I^fHjy%&R{YZkVvyA%uuA%rf+E$cX53cV7ysZFSA%TQdKIOmj)8a^>XRCbQZ%PQ6PkNa0XaoEnoHUNRq#d@G1bk#&$DU$Gh>1ieQsqFwRRLcRk ztb1Y)H?F*<3lkzOKiFtzgm82?8re}J#+wUY8f~uJNGW9j8{%N!`$1N23SlGjE8;e{ z+&X^xqA_k|pbH7EXt@5_d^~Fs)NvE)c?tF0)T#qghTuK=u3f1Mj$NgL3c7=GR(!yp z0R!u}kfMxX>m>s!6i5<4iO+!)<^|a-9W;B0$cwm%E4`X-dy5q3b=g&gFoAJi2~fk-yHswG%O}l&vhh8?MTLn74gotvi)V3waYU zaMONgZ*;HsdA5S?FjpI2vqN1B>=gNi4(4*E;J`Qc99C!;o;Xwpgi|d*A8nG;{y+sL zzj>v^3K>=6t&wZ;R!lcyaeF*B{pM^|-RZk&naEy(0mC3Ru#jkIokW?6(zU5}n-!1F z!GTPg6HhTu+sgf=)B?+X-mEw^7hg&Gd=@J={pU=D+r?Iz4`>V5)RKu}I|<5_7!x|$ zk@>M(f-m1#29ZlEeF=$;rTtn`C^CumzR-wkvR$m}pP!mFmU>x8_{>rcQ-{7Zc0>N6 zr}O$BT#bMKnA)lxEnkA+HW^gHvkAM>b${em>RLa$H>ZR7duPQ9o-qrhZdn=WNr#v6 z&!4~1uHM=VwStpac6PN5U3CQ~9(Q+Krt%fcl)`&3lI=@i^Z4_k)@h5f-bvMSwbmf6 zH!jjxDRAQ>!TjjCsoufU6Yy<*-8@q*RRP}*1nb^mXssKT2@Ff3BSI;g(JDccKT;NZ zULDg#TWkV2g@(?E$`4zRa01j(&R@f!ZH7-A+`YUS)~=+{F{lr= z@uyp{1mZDvSt*psTCr=s&Lji2ywxi9>z?W}Q?*4mZB1;~oD`2@MXdb|VV72t6F4em z`|P(Ey;nPQoJ^DGX$}}7467Y~?RDpL{=Kd?lrt1D)42oA&u)N)C~_5JS9<^iyg+WF z7Z>3G>ship+^ylD_5Sqo!#%}J-KDEN=Of^lwU`~w@vRH5((S+(0kJhS6l?1633Nwb zTr91!97cph^@)%a)L&EjeT*L&vze^B4EDMe_A-!EPP&m?V=FBFjIhh{||JM?m$Z^5owb7wx#e0 zWXN_^@Hxqt-W3*fG$>6=_Mlm?vw)<6xtGOUYMi@5TK&Z9170mWFBTW~_gy#0=e$Bq z3xxSz>+7KIzNM0aT#){q*sM@bS=psDpK>g0scX|Cf=nx~V4a#)|4eO5d66Ykv}W1> zBh&BJTmBCZh9v`WmB6!K9B+0nb6bwWaa(8P(7%2|ugZf`i9c-VteN|v=L-qjHpioL zDnK)THFE}PbB|U<%QEB(Zaw$VV#UGWLao!&B6PnIaai68Fv{tNPujbkSWHIuN&7r2 z_uUDpk3XGA2Y&TXP;1-`cN1G}CXSA3pQdFDcb(3JcH2LINDolxP7ZnGjO~YK8TY4J z{&^t=K90w$VbJ~rO)QE(cV=R2EVazQ?Y{JjGGY_*>`pvBSxQ^7-n3#l&s4|Ybounx zcJ0-FCm=oi-7o10e{tSip;^R}z=-W14!MsCJp#CcZLJxXttbpivJxe*UZ$NA5-bEf z|3anG2eMX`&Fbf6qn6sT=XQ0(vDsmmBs}T6ppc8#L>a?JRwB5jnNH|Ub7qgl=3AHO zY8#N5ElYkj->raUNT!OXyZs1&8wU1tMwckqPuv{=SLQgzol1Vf9leJpwO_38uev@5 z;_5@KV2jS|mzO(e66WjSxYmDD;p{sG>(R?-TH>6n>flB08L76O>-44{tFgwlJ3Y)CbpEYnrG zmB=8R!+3Teje>*MG*U=26V@&NCe1OAIX=vyG=9lwNP|H>-%Cf_CO>h#%KAxpp#Y+g z)qrW*#kShTUM}}jgyYp!VVKt{@xBNu`n*LSL_~R2M2L zZh6&bhG8QuUw&9-&XZNywU}re6{Ty5$-D zRuL<7GeCIQxIW=8mkUH8zWQ54NQC*3 zm9;;+$SPy#HGZ`WvBY7IkniE&(`)W&J=T;7?G3dtR(h`RIYyZV%&QvHIl!&q%#H

Brdn!}5~4h)Jd2M+55rOGbB{Pv2h{uvT`Nxx zk8WVTuq=J2og0{?wMu1pew|19N@sDEx?u5w_ zwX}#vnP!vmv{w`BDpf(_!dRqa^~t3)-Lr(AoUqB5bS#MojWF2W zYpH@Y*i%hxqJf+jWyWqS=Kk?A8iKpR3x?G9v)~{Oe**o8- zO+%_;x8USn?@>Bs-_;X6|M49Cmlo@{vIY;w5ATNin`-yAlRLr^Mm0MNG1e9DGNYjH zWWl`_qOq$JMsz8lR;P#??de!ZuTooXjmL}32;Z8VwR)xX(t# z`%tq*NL<>!Qjfi{=3qslW|%%|adFLU@s|QyZ)5KgeRs-E=)h2A6%(*2pt{i=OI$lF z1HzW*qx#KO)EhJ=n#jRX4x3HV#zU*vX3nOeSHK~AZ8FKC9tL@tz|qA55a9u*XNVY{ znyj+`Su`zhRUPLej3_^G<3=W3@Xf~k7V~?GiyP_tF3eYOnm%T5Il>A2oUa0qun3NU zXA>-`sqvaChFuPw8#0Jfj1Bmx^$5M;6SgjfEvN75xcSABFcfeqyD>p&)~5?UO<%?S zZ&bj;(~HUaTa~@aYAq34LUnu`$XDkVRy%uNs2!CU-34O%X$=bDhz0q68BLY96znY# zy^R*}3%%tqwugb4LZk|qIM4DXyzsqHYSR<)T`P2-+9x`Zrms*+iTP9K54)gRu8j0r zwSF{H>OzICzMdKygc;f*p<F;*AwHT zjL@j`^O~Pp;-{eRXm&X)%TC8`c{*tTAPS#`>ujgsy19Cw{=BAJHJO?t*7`ZsYF^eX z4|JT_>+_whnc2Tk?KyAPLkZ!R=rBvB{92UHaHxI%<>KTSIzOD;;LNsx{6}uKWx1+v zAzX{@ougIctO?c**jn{v{}%DnJweIAFDpbV6uPwJuBkdnsn^e>HsIi_u8M-1aV2_7 zB-6M4;jJpeY3o&QluOjDEHA-+>MGq?eOrY;G0KV0_G2YG-e0xhKgR?Q#`W&C@In#! zuLAw6So9}wt?6V*rpSnnXMe_Fav9K%@10dvzo0B(R8Hd%dCc#s;3bcb*J@L2Q<`o@ z#wk2Ky%iNZOxtxNFz*S|MwK;9cL|r<+ayl*kJWMHlS#bTeBM3^HY1q!y0A~Gx6hiZ zt2`AXS=&(%DrDlba{H1f-^VnD8Sj(m$&lLK_JBWLXpglKU;n;DX1;TtQlW}(J}vu&jaRzT7k~dyJ1z&eV$YK&a=AV zdArs_aen|8b6Gw+_dq{YUc}Lwjg9EJpe4Wl=aj?->>HY?7g2qhxUwtexkJ4?Sj8pr z6D9nM+^=V$uQhaSouwUuo=8=skHwqB4ksih*r8%T+bbvg%Ltd9wlJryw!bph=%(6i zC()&TXBwuvJ%dTluD36O=OGy?Pp&q)W2@khOD$7S9#r(+vmim}`O^h*OC5$cwght9 zq=ruBxC1ODs-5kJ$RN10b=3K3(^*s2AAD3h{rMl#TA<>4&7s!Orgg}acjHIpLq~rN z1Q-D-Z`yuSk6K6|*(b+*lsp&x(9Fe7Ts?n)PD9jUc8q~AT~%aEXq1r{i`8?!F!s88 zmMP7dfvE<#p(z(cRbC5OxyHjKi?yRWR7EI-zq^}7O<5Y`6O)DS=;t3nKc1s9gXA+) zi0&~ykD2)bYiVTp`aZDtiZ~O$_4aiimIg{T2MVOkyFYS*uKQOP_80TOHXMIEbpFEy z@c_C_<tDYW?YC6gW74ju z4b+h$JThgzOK0&X5NjLN>L?DwnAwuVQYTsXv)K2%!|5hUuqGYS1!d)lYWt&T4Exe` z{V`=KFL}EhL?)qZM^?+I)=zPUk9(z;v%*LIx>}EZ`%hUG^=3qPU{brRe0S>vI{|^% zv7@6(X__$^5ox}}%iAH=KHCD_*Kmk<2#_fyxQEDEwPb7g6S*2h*R@I{V0_Qn13#Bv z$z-0siY9SnWsoW6tS@0a=a`PA8-<>ZOgefPZL^Yiyw#(5Ow}2qF_l)LrD-q= zZk=}CK~eIb`q3SNasa&E4d`Mb!{(dsfS@bf<2J*ppE>~k=NTQkrR#8U8!!UU9A~1d zC)dz~8FoVzT(oLVMI!SUs71GFaeR-E7Ex~{K6CqCF>dgt=28_s?azELrTTI~pBSvH zQ^uAEg>`Ht1~cDy2}h?xz(i&cwwUW{1YLtuek0hriuM%TEj=^IxSst5cS(DRSR140eM8Sz ziK=PD&Oq4w?Lc6uwtPVn8+A463or$R?)dX0L|r?GS?Hp<)X`FiyTgRDX}*!OUwcDc zGu?x+`_sBoPIUavbE&DD`5Eao|3K@R6V9Ou->QELHdLv+sGr-T*N1p?sDx_X9nf95 z3DEVAU2n#CE&`P9tI1(r@sNd7RcP+Kem}iyJc1kH_`N-81zSRKUc;159j?HWSjb)V z#OR4l3yxr?XP#*GlDbMq8x~`5vP{*EF)8f%rc0Wrj-bINxO@-3^xX+0q8H0nTa#I^ ze_%?gFz8q;K)k7y{n^@G;Pmqh{Pv!4?qhrrm!Bx4A#f+5ufz2j|)O&9CVFI*wr>7YuE|3dp?Xvr6ierhrin zNtJ53*s~23j(1f%T$W|~?7x1Cp9{({JB29}?H9mVlOCDvH`n?YxXLIfX zKpNLB3|9N>9E=}Jp&K5u_>k*hpDF|JpzP?=fFh2}CtDJ)uO%=Hzg!)x4jVy~CrY}$ zpBUPB9n_Wd;|rU&pZ1kvR!Gzpixf^${!)BcIj5_xw#;EBbC02IkO7aaO z6>Gu;%VrQ&+#OSTBObo*E8E7~KjDx(vqs}_gO4CK90~)X0wTZ15U231SnH`5u1d;E zMK`hj;E9)jA?w&_Lz$S^2bXx|*GhSnqj}wXnb2YV&H%O>fGrFiqOgghYcjJD?F|)}e{Z~a2rxFgat^1Jhgyla75Tl^ zd|``O*mS@yh#eXvTzL7W;hh^>(KPOe^n`HE1(6J1$i#xmqG`SrJ6k}o?u+rEO(oVV z*H`x|J9riK3D>ql?Ux2&tCstY`E7UZ3_=^>qWajK?wqm9tP-*C<~vVf9L3M&0qm)7 z|5>g7L$O5tx#5q>r$@O)`jwTWMS<r=5oyK4UZ{m#Uy#`@O#dukjBow*y2@Vt^?#l~t> zmVd>Jww!b7%+p(jHC=Sqt52L*XX18mm;sKihC@?1cS!GYr+I}K{X{Wpok zdh?7pYGnZT7-rL0XTH$^yZ|;|LAhjyab<}yYSC_7(V+aS*^4mP=t$^+Bd9Sr%;R&u z)R2|y8}3ZPmvy|3JgU4AucNPJv>4TvNt1*_u5Gz1zQZV}DIqcXkdw16O6-WSWj;>) zFoLz4#oS%v*UyT+{_Pn}Zl8}?^RT|Q-;l5U>P?USXPfCj0wRjqr%B*moZ{l*2OcHr z1Ix%%ndF`?yOuVsjNK)yTJdX^s)SA19-7ij%UOZCdHEAf6U~F?2?g;h!Zw3r7d2mU zr*1d56^e!o43M{%Gkc<|spk3B9PC;g3tY~$e648Q0GDFZf!Mi|>KVFlu;D<1XuYM+R1*nLM9~HukW{Tb= zBgSUC%4kEz6lXWC^LD%LU9*>@`=I_B2RCF;*^U@k2|IFCPGR%puy7s31=np%TNN+I zd35pSKTNv+t_Q!p!kE084XD8$NtO*Ip$=1X8UA`oXt8~cqt$=iXL*@-{^;Qd56(9k zc%Aa^w!hiM%TTDP*-3TP%=cD}#IcYlYT6JnW^!Tg8>GkQTu*cH%}kR`_>!bcup3Z$ z=eM(o)vA>68hw&P9`67+0iUnO;6)dz!+}zx{GZN_vu@*@d6~AiQLm zCS9PMil`*d%t3X_I=$L_nUb_cB!I#~a`u9A0JNYVwVsLpbXyYAOgPmx> zv+{=V1Ylv>1^BRBdvEY^WhsB(0s`(Xp>^lx6WV~h%D}JkmkbL>>ippP;MZ3>>~FX7 z72U}J<_(FfhkJkuFKcQj1CxkRw`oOc_#h3U^4^P%bJC(pT*tSlmKw^{(a@(SfY`ryCWkBnt_HgSfEN zpKI^4evPM=iBlOEuEWvwu#w^6zn>@+?sMr{^m=00F}ETeI)Q}~%3rF|!Qvy8yPdj{ z7Jkii1jo{0_?QJ=u?ytD7)P$oYstYfwVa+L zb9=rKv#Pw{+P~Q&>9KX-@#1W3?&x^8_RudI5RI|9ZkW%``Q-u!X1CUO`4Yvy&6SWP zku~gxCiT})VEC6b)1O*SsCsy3an7NDnrxFocA`&RQmTLbCUQQvfC22!a`#)Zjgc?t!iWti?c8i>1bHLzA~|FvAt@@>UKy=Uh~ zHHA37L@lNfSbpo>qn%|nJM9J+;F};%2tZM{0MyKR@qu}LU^NYGNNi363oBg%sTI#` z$4B`WZpypGL9qiV0`Gd$L?{&1 zlKkr!7&8=u>vH3-6gR?FlvS!YZdBMr6HnPohGO?)pF^wqZ_kJMc!IDduKB;RIE_i>A)dA+*KZsY8JR(s0kOHC$9T@9tU)lpu<%@>zEW zz%?Fl#q-yO-Ms-YDcvK9sl^IcU_6YBEWde0Vin+SNAW()WUqVvySQLX=b3ygl@`q; zp*kf`4VYJehf!PLd4uk*%KVEfO5svKR)0nu&2*%8o)HzMcB@-`tae(rBRC-Zu=4NH zaTmOX$D9NW2?Zb#YFFr{fuG)!-Ey|G@hIW`#M*bYd^{&8u$9uxzkU;{^j4FofT!9Y zqGIl8-%C<<_=B|P{L^MmXm3&q?tgAmuP{D5UJyE6_s#XDk}*{0N1@v2?qt6Ci3Q#J zRMgm;^oH-4rQKfU`G+Lq^f;h~6D}h6JaK&$$Nl~8qL1+(Tpf^KpR|s)MALHp_s0lT zuF0PBDgC@Xk?d5&Sgp6;?7x+}5Dz*3_0iW%|MAgY|K}~6qiIEzgp{+vy7asn98Vxp z<7^)wv3VJNU{!9wYFLNlf8XzZy%P@7 zGG1%;yLgLszV%xoVV4eS$@tfA7%i*ovlT{o1x6q8_SI_X?}|`Wo5!YUrZy?6G6mSG zHJKQl-5fGO+0f28DQrd>Gr4k8KHPB&bK`h0I3>>7{*=)bAIVILHu}%*d$b8S!FDpWvet5lD|12c3(uq0B@0 zY_(K8fCGz_mvDpWdFXfTL~>H)e0D|3e9W$H@QH9AV-|$5TQwFXTS#T{vt#QYW)?3W z65J=2ce9JIe~WC|#^hBeeoo@?AQJa&WMh7(lx!^z{A*NBgc)UKU}FC9%ke%a)Ox3J zl9(-s?wTWJt|@J!41!S6+^NIF;Z41nid~Ci(+457E)p%3tlKao6{)8guU8^u^r{{8 zl(TILwA{EJo>ov|pjRuf9`}cp7eLO(Rib{zk}I0Yes+FthkOl`tx6A4^PbLNKs+C< z@}(KwR|4h=?Z^GMr$A?UC7)z_TUO`?_}ATu0P*pDzUcR}(6IZX+2*!fJYI}xV8!?z zU$i4t*Y%?_!t?I@58pA}RNFUdM`5ZA^gcIFp1i##Y?U&gUl@oO7Tzw$uRlw|Cur)A z)g+$GXF5s`Kb3LRpUl1Tl*Gn$@PE6Y{r$};iHLgMU(eoe_EZUsWVB)H;(?_kFcy0f zgll`_3MDgOhe6W-w@s`DO*$r;%nz+TxcGhMmI zk0}Xujuu9Z?s&QqHdU+^0{S(;DkI2?@?_Ef!TlF6QlFpA9#p`xLR@y!_k23CsS4dZ zaTTP_^e?rn0{oI`Lw#fg@2WqV+!ggb9`UAJ1^+^QHV$v%Bu6vETjEEHljAe~)Ew-x zjxakvd*+eVUI9^OY{ywE_7lE|6KreghH6D+a&wFjIJLK6i(E01&0S6}xYKr-Sv{#W z%&+?F)t}0LU8x8E88ky$e^$hrpnjLk%^TmG7}hYShq(ID#L`k;v6&yRkuzP3S&QCcUXBDDk5j6be{8WKGTI( zAyP6Aa(8~+L;%iqNv!1tRAxOLJ27avUdtSycd@3UmCUNEmU^DG3><9!)tx}ruFm$i zaAhhFbZ0?TJ+?lXL{;8OhW?G+Yo2q7cvr%?(o}MZ@9t}eU+DgeMae!?=y_b3uFy-t zg0dAd|KvQESco|eccbj@gA;~kE_^{7kWJiEzk%A2G51oJp@LifKoIk&ez}@oIORr! z`qB>ukcg&lD3|`AWR%SI)BkTz_&0PS&cC13e;G_d&YNnq^h5#nt|b(2UGlL|Rtv@E zqF~+zN0PwXC^-d6MST;#Pc99WFGo~5q^nA@RegCWU}9p9(cdJ;lteq!#w5f_w3$X5 zLm?Rqgo_D3epW?HUE*MJIApvTa?+jTSj!a}H1m>|#$=LzngH3gz9^}--m2PM;j4`z zH{7SJQS>pDV_C^p5lQH(Ux|5xTuqNrpE;6!{bEMRYT&6cZ~@Z8q{zd?rM2d;JPbEG z+5poE=A?M$;}VX!(IB@$pSGe>jl_b1$BxPg8F=FD>tb*a|LVS+nXNaA(rR5ZXh~G+V6=>bGD0jcy+J~XpNw% zNgzgzp!mV60Ec)hIf6y|BTR>%!|I|7u{8-E1xtn@nb%5AM#p^a5lIAW4J^Kc z`uW&9iN>|wH7z^_3A26K#eUf_nz3Mg1x0080+xi>M`IJFj6Rd>0yTKvBM4@%NAJ@s zd8(kWAWsaJvS++JcLxK!cYf+oC5dM7R?DYk_znOF0A-bOoLChGv&fe7kBKoW6Vetc z!C#}{Z2$Mtf9nRX?BgrUEc7c7_;c>RDeHC)`Mz7$lqY}9f2t^KvtrAHb*>+Dj3s=k zoG!GqP5;q$ntY?-$hAJcsG(!3xAKsE@aPIThg(#)Bt6OX5v8+XF3gY&b3ICI&R070 z$u9@Fd?Z%W2qYtllNzytjO6gD5v516($Vs&Y3Nb~D3D%lEc$xs@zGBB#M~2BYz_gk z-(!oj3pWSAnBZT(J#DPzsz4{=P-RJGHhunu=W^a!&n^P!_;5JK+;$R9#SU5^QE2a9 zLN_c0ypq^0zu>16+Ez&A=npO#bAoBf*@u=k)PC$alpXqCVCU&{&s51j^G2wD$vb*^ zw51}J9-_@SV@5C^f#2wpNv_EDQv3y6l&qjIbV^qMlJQR}rw!?vRR z`s!f@$<;b5)$D)s2}z93zHhnxsKhZ!SFmkrP*qJ{mF}vS@n>k{^$a?_cDCYk1>f>$ zuBjIEg6{(LA-!?V2kJVFBny%R=E?@wN|I%=;6@m^cDwfFTR64X$~_-X+=9+g%Rm!Q zSvcCuB{+4qmopCzL1>P3?nKy9?#-PY8s{>%8$h~gP5ULG*P7e5_o8-{sPc;yVD{BF zl|^kT02fW=q!VT*&f8#q3M2%rM@#PyyVj-H@1PA zLy$1NY@*pgPyFp!CT@GOY4v)h!QTAky*UX>ixQEC?gF*HsgNmKKV9s*a#A16R>jKa zgk^n;yTe#`veQ7RMAOS6mANG>s-4eZQK~O&ePC5#l2P$Z!|%mDrhu|i&r%;>==!-P z+0d1Ugdf|b!c>F6{`}XcZ|dJ&y|dUNN%rVv?sYT%u}VW_M@z&?L{rm(gOByHKXLr% zu%=3lD2^Y_2*eedT%Bqq=n}%DnqkjI;v#bCCvxfc##M9y?ewmSs>!dohx-rbS=;}6 z!CzjoEMf9g7c$GukVrm!_7=~}gwi^B&J6QK4&_UiKqFpFd$|t5e!QTOae^}WC-J9m zVl8QTY3XP=SUwxyHZgPbvJ3KY4@VPF1pm-5M#j^%~ z?SY^ETp9%U3rLE?)zgd1>*W&=d#QUJwV}NS_43y&8SF-lCTcfi2W{vEWBKI87-Wi=~%w$ho zie(0bvQkWdX?b<{&CH-&HWmuVLN5Tj2#R85FX(%Xm9?v+TYZVWgE>;Cb-E!cn2#z|Js+8C7uKHa|T3w1M-#O28` zLo{j@X6_ZFelbe^YVXwW`qxW;)2BK0!&be|ZQv`bx-!yAI*^NA= zfpFR7MdfLA%DBc!lWb2?HlZ#uW#x}MGBbgN^hgkEmIm3i$3E~#nl-zP*04x_40PxC zlw%?sV9@FG3q*s9YFCV}?%Y`g(FIOBVDlZXJqM5jdbc3omz&M#uEo|8boa(~1QE^NJj;C5roy~mmQYQTNeX4?8nofq&%v7*w&H4m4XPJeG>2TlR z-oTCH#Jm{j+&Roos2N8hDIT=1CE?KqokInA^ydG)82?56`L9bqy#)>nUf*Olr!E_9 zMbX`REoSl5@Bp9BHB#zluV9P?ILOypZ0{jI>PPq)n&Zk|V=tH+TH3KSqU0xeqy}9i z4xk3==fUU)-2K`d~*9&l7gv!%wY>6i6q45 z`<9w7G&^7XOAhU*lSg4+43y3=RPA9eONJq%tJ%$_Y|mHmuVj(AMz6lhqnz50?GHJ5 zII?rJJ*DK?i>!#$eTfG0j7qYjV6AGIFw@Q+hvGs0biv4Hl8y-MAP?=BGY0L&BX0q& zGE?KBq~wbA%e1=u(p;Q7mVY0t?zm^=Pnusd9|6l8u>O&d&B}eTma$< zV&Lw84i>sQV-7L_+tVS$(e7!Y<N2Wf(-1{>fUzV_U111mvszv{qiD@8W zyC;X9^lsFluQ(=R^wO$?SUZ8q*UMCsQ)Namh*PTCc&-+VD`vnWSMlV9B~57`|G^7J zK$SZUg@Gl)ZHp&+b$X)L67L4O{c{}`OMIRK_2kS{ z?e4EP4Lkclh#6!c8R8~_=7|&HKWz^|qWcb}Mmvb+kbQEXAAw|PofbUU=)6;`xr{TP znXw~+1_F3*4Z*6(X2cJQ9~CL&QF49AM!Je|yvePEGy=9VZShTxwXqz{zZZes{j;n{P&MKO!3 zQ*p3zk2dw*)Br{VV1S#bnmd08v`a?&nqQ-uJ;~5bZ~!rP($?HETl2p09{-kWmnHWA zKvPFbWoP-Gv9Fw%8s0~~of%k^IY<8Nr`t8gS%23u^A)=j`DWTks+{1Yv&$kaqbyZD z{3e$MP$x*j*q$y43j>7Th7g_^-F%tW!$%bIrYVVjxE6OHa+%%)uYXe8zmDHuHu!&R z%~KywkE7R&uu_zUWi~8n7WOr+ zL`KWMz8srZ;~=9PAE?yjBIfm><>|Gm%CH)^)&4PxYUipwn%B`PRO&2^9#Aam_tHum zyQAaH`Fq!qD>ss}y=07BFoJxhqm>)16o!-*ro>xm#lI$Hj+>v>wEh4C^Nd(Z*GyRT zOH2@9&&kl5#(wP?o7Y-i(`@v{~4vzLlhR(FYPj75Sk zj>Z_I2%yDTi`a0Japux(P8+#W^YW3!_M4SjXUq@|-GoW{)PUv~eDW1XZfdS)*o@N2 zVr9)Yt3~+!Q^)W;M^7kp9L}^CE#(3iyGQN%tpJ<#WoC11xqpk#A3yxa+B^WZcK(~| z7PMyJPDX4+Z6EeMG3Kd}QD!L0s52yu6GB}PPn>N)O~t{jBqcx$Q8>*~d>VC$4L$1K z4~QQtN-ruEDZ7QwSP0a8{@^r`^+k^Ix1@hqJusHMRNoJY^gD&>NeMZ4OB<5Pu{RE| zgrj733ME+jm1x~VA{@Duhc4 zB*&=5s|3#<25c+Uv&O&!v0t+@Uu4CO0BH}<9ds7DFBkd=%Y9zDWN$Evmf#z|I255R z0B*QtA>Qen)@ZFk!v{z=q@cd3_ADj6=;GoZE6sv8fRMTQC6E^m?Zh}~eS`H9)Ku57 zuyPKl)-r&&ZIl&BY;mIc>xMVgoAD#lbA{xFPDZxEeI`V@z5EGVF9dGv$_o%{sr>j` zOaEZx{*4DEY(D+Acg9zN2>|(8^94lS6XF)`xd?Q0xriatdmuKs6RZ(NK>U_uC$Y!2 zm|h#J+zMdDc*uHztXVNSR(`Xq{_NZ}HVHV6uewT1Gojle9|4(3KW{7$n&V9KjLD$n zdeG5AYad7`Bc-^J_I3rkeZaig85Q-)iS^k}OTV?UeXI`Tl?29d69-8G04A;*^Rn=J zfpdFD8}(L6)p2=;D+MLFjMV?nX*Ftep9@!D|8620?*GUgqd>00%<9KIE|!`&3-f0K z8`^j!=9AKtB<|}s_gVK3OtyqD5SGObgB*D)EN7!KLK;z}Gb0X7npv*OKDs2QnJ=Cm z@%=NCOHuF1qZRIN2g<;HS!N9{&GM8QroGD#DKL-24zG-hbIvB#GEiMcrMwh4q}f!+ zTuqf?Y{PPZP46fxUCNW^OQ#K6EZE&l4mZZGJ+%Dmr9Axe!L^<YBKa;h8yYN~cYtL!FR$Ocss`Q;d0Xjb zGCC1G6<(4%_=k6n}edDWgpImQdP_D*K)0$cRv8hl`9>8doo{%S#;6D z!@P4T?muRSC^JM`r(cJP8Gz_1>LV)+ru@I4E7jzw7pM&+%3(0Lntf>K_<{h4d_itW zN<@c47t0p|S46QFc|iGoW{CBIF^>R1PzB2_!kqgdRU zRvdr^y|Uw&yf}jfX$+SCj^>%;m}u&@yPHq?h3$S%qyuRa>8H=LOR-;l%bGfm?T%_d zw*q73XIjg9L9Ttf8V$f0HI!xWKxm=<=vR?JEBxZm*H$W-GMM4iFD$jVf)GIaUpYbC ziNB{1Ld??Ax4`GmIVhf@Sw)5a+MjISF-%_sMyGit)}|xXe^x!vaEvAWUmer`_vIna zU}f+TZWQY`AN_z}onUdbiFtOnop+lgPTL!L&)A?{Jw({4l!kPw;Z?yz5b*4{LNR4) z`vWl<%kQTJ{9^#8JhH;2k~7}hJUBg@x3?#Ojbx+lz|z1(tKZV}86#dL(>%EQPPyr% zoG*p#j-U6RxQ<+x@2$fCWA(zrqE)%#wHB%`+o{pBuwN$;z4)9IlgwrMR@7J6Z13^$ zT*Ta|6}d6Jn9_X0FNsqXWoZ{Z*JWgzs-t{gG79>T5xJ{wjuDN)p7nagtu>BWbU)Kz zU-d2J<(a;jY}q>_{!Y=C)Ad(u@yP!aZXHt8rZW4|w;@Q-!@ODnPbm(AXZQw;%bvCN znu^2@qG~?T_(Go(H7TBJYXN#e$FhfjJ9xfIyQcLVtdtbND|A^df06OdJAzH$Qx-{7 zwbHaujm-b^L6*b50)KB=-C$?2Yc(a%v@)kR9+?>mE6u1nszodINzANFYaBM9k9E#T zrrRKs1^SRsq&Y5o0<&#NSwHz=_m_~!mY?w_SG)Rdc@}vk8WzNC$vUQ0FvW-Z5Sv0< z%M2GvWi$Jt9Vr=)E7Ccjk*Ahny`H*lQm|J%EO9Km*T2dm0dE~cQMiT$oPe)m27g;^ zLy)!(zi1cKD3Sgk@s{nc0?BU}-PfAcL9&fp@An@JlU@f}53xfj8L5@`Fhd9z1hf~b z#)da;mBp>tm2HZ(x2c)1oOusw0`gi%2<%rU^s9rGUaR-5lIzN8d`&xhCT924ol2No zX6=eW-8r-M+Hqt7{Ujie-SP-o;CjjtXrZsoPpAv+rVm)!TB=$Pp!)9UlHJZ!l4%v7 zyJG(koz_l!9WJV`Y@l<0eL>zW>x-YPI{a9&K1*IY)##KXJn%=yU}nI2X6I~@bU&_- zeh)l|Ye5^wrdB;2CK>!LEJLxw)b1@G4;4G-wxmQxy)*=xpeh@Y$aQGH!&-_!+L8vM zij{7eyrN&e#@LhoyYu4zZ(?gA&}rj(j7SytU?L^e?toi#a^G~DS7a*cLx`H3xAX?K zvl>GFealea#Bg8Crd_?b>D6K}MVI?74`4%sEUDntkCip@XR|Y3-h}C7ieRNB_>?8xBb&l${vj0viQlN*%(;Jq*mLN zflvKI;lO+V6fiyjitG|0Yu(vzSAq5-iU=vHe7o%RfVqG-N#o+<@p6v&#;QK!H*77Q zT`7~qd~n+;mKYU*5XHr4^-ZGGs691HPWQ$sjjlVl%|1>8lk7oKqf>j!DX9YOc*L+tjg*0Xuz`5a&oC5 zS!!H-2kY12`Z;ag52<*P{>vZ=L{;g#exM&0Hx=zuKqSg#Der9|ucv1ll@$1{nYSNk zgr}+rS*g8KOC&CuqS_rhdbmrW_1A;j7()t73-8j-oy?rozY4H+rJF|lQCgHtn7})! z8zVi*6s@E>r1F}!mV<~DDk(&S6P(Rb`IgNyQ8!h-J-Yo5ohBIOxVtp;)w{wXi)mRR z!$KKu$6~d9L3xW>?;htU%WwHeMi)aa+Z*N#im!qMuY8rXTlKF_Sgo3%^&`8UyhK44 zf9=c%sf}ii;pwg@$p!6}kO(X8r*DbrW*tEhp$?0|;m}V*Rce8w*ze1nKPFMGt^P1+ zO8nATym=BfDJdg_l}l*1-uJ#nfB6OYbok(&P0|w6Z)>EIOTD&(_w2(vuUI02lau|5 zi?OxLbZ;Nqa$v6cvc@aPKBg>c(Ungp$GBI?Zk znTB}EOgugcK`1!$i1Vq+Erx3XQ{DII%fuHUW*; zan2{~jrt>tZE2pFboF=^vfs8FTE^#<=3UGEs<#P3L!zLd3iZd zUI7>ug>(0W-R=aAPYM7K?9B`Uivk_c9k$j}aU28qbBSKs@K4;5#x~j zY-HiCa?a3r9udw9M=on|pPMfubIy(VdMptL@08koIM{RTPAW1>@)GZgk$%H^nzUDI zAx2O8Y=zQj5s9VHx(O@K)czcHW*fHJgMoW5saA5*bn#{TtF@Fz z1mnXxqYK%{MJL%(qkCY^O{^F@gqo2gF*K>+DXQ3T`S7vvYa$Exm}JwDxA(cspn z-Kwqpsi;M+{F|*gx@GcQ_hNkC^)_#OcO}_0gp=J~tg9hpzoFE6sXWoqX?ML&$(-G? zRe(E1+WfdN2;xZ>69JW9Ke6Bq_Zin<5Abw2`i%RWux>sgFe=!$ma=D2Mm-oiE{+Im zhu{@S;E+Q?<(03t>me7s$7Ndul1~4EsmCEf2k4Vb$aAN0FMw- zw3~D#BNb^A-ZI9g5mb5-GE`saFZDji6Y5|mCX8TjP{Uk(B_3~sZIwpt>XJUK>Viq< zI%H$seuaRezW({(5AY?mzu^bP(LQ@}qkUIgSEH=NM!m3}VMc*EXsV zW$=0-cY2+Vrf7h7SG$(BjLmQEh-dAXML4?2f?~+I@vj$xHFBeCJd|s?mvnSvVL4Gz z82s&)u5_nRlA1uimNT=ThAno2{V{QyOimGh?Dx+YH22H5GPo`BgB6foEscn73E=_)w_zDJm?2>YBer^kx@;UCMjncuL7=~R@!q!jdb3@}^z+r? zC3%xW<=gbqFX)h;JasF-X@ABa&CuT0d}kVW3veSNjrPu6Ezvc4A)#d%6wuKC@Hr&s zP}s)0uvAxUsM2QA%zFZsCp#zJXoqf3GWMjC$JFClog^DPjYHSyo^&-0E6r+nktvt1 zR!&FSV=&og5=p|SEWNJ*E>?{QbQzMoAqnm{>PUbctS5DWKnM$ahlX@Tydp{AGoGX2 zpqE`0eSZfyhC354N?%;38Dj98QG!*ABx27>dhqGWtYN~j9iF5R$_^D@B6wi_#tSJf z$?|G^C#sE1|(3egLVpFgbr!Mah`cE!t;}yR&UGKn7X;K{*O-&2N|{~!X%#x1;%((`IUboL zDsoB2b1-h0a+ii9;*F|dADst1ajHj<@3(^WNx+rW>m|qo<=DcpR3^oMu|*nBi-Lo+ zq>|$hzUnW3?Y8It3z9^Rdq?$>etQN&Zxs_BlPL_j$*Zk->h{Brvd8NY(*4A-CQC}b zyVt~LXN4xUM5vTJ^MnW1cQIJUI%7OMjBFd+b}AO(f7R70>3y=l=9yf*?=a1p)^yjZ zg%0kB$eLwYX?*o^0Yb+v^DNiao7(ys{}y1D(Atm2eXw338yyVp2>EPz#{$I1r_-p6 zDuMGK`qY3I)@+NGLkEq0$AjUPnG!v~PU^I83nQR9Jp~8@!PswtZqifBv$CG+eb2oQLC-)WUC;!n?q!rKAIa0Af%R178s}eL% zU6=yNw&Lt1)@d61B|P%oCQUestgk(XAM^5U+Gk7Gcgn+#YV2}T+b&0OwF2ACF?1x` z0!v;Q8j=c#7Fm0;h_S`q*9!&s>RHTgH#22-Z?^=!)d&s4RmH_VC+3rDa0cl})-=CT zRaUurEM_IqxQ^t|@}UmnbdI>7s_h9KIL(!G9PFftl*n^B7>`8{@+73zTzfs1`OfL@ZXZ=g!@x(>rmiFGQXha8 z?N=6JZi9!o4;pNE^XZtEOinRk7_7aDqCy?+2g6fnlBs2M#Yq%kOw1%re?IvBX)7dS zv^dVfly5ziM~Qbt#!7SG&_}tLgb~}?=3VP~J&}4kN#qXu9%-i`adEg0$hCU0V61oi z273b|3?iNoN6SD(lNSd3l!%LKVPj*sE!Z7cj+1}+%WFTP_e-3fl?X{=S#|539Ga4w zB9WZ3EGhn>=~|xIgR^Smg4grrrHJt2l#{^<7gOTy>E_a)`5bRx{92bz922>H!9=xzxMPU)7AtXx)p0lr?mntT^vK6(c#VLGCZ)jmdr zY)pg&nDEzMW{&QR-{->{AoR+CoIM{;gWzkNaliKa(@d%>7n1RHyDP*^%YK6qesh)J zIm{cZ7OPl7d>*&&C`9p0@3y^^%J|Ti@!_u&=eI}cr)6wpT<#2*MB-xNoa-!jyNJ;7 zcMWD!2wlPJf?~E`>Y^O~d{8x-$A25)&3yG%8(c#xVXPfiZaZw@poOu+RhCbs2$y6# z%~v7$qOrL{Vk4CW$s}TZQex%N$Kxz6YGqgFzeH-b#Vh#Aio^B%+VHY8GG}F%@Knh`SI+Ixm zR7j=DHR1yigEZ$*%Ex_o)V;^H~f z0#GR)5d51No_SYMZ)HtZa&&a)i=bk1o@YNMv7gi%>@DBjEg3B^AxAIH?3phYVh#j> zQgGDSL~x5heOy-9Pnj6EraPaACFDBbC?}W>CIrkEOSWY>j_t|=^;3e6`R#NaK!1YILcB*=0+C-gTSJIh08xw;|T<1}?b>+g~ zEW3&?K_rxS^vlDA)N%7eFQX59ec_TmH$Nmz2e)ilaGBKzW>pp8`K=)G{aDpy;|!j? z?>A~$IcuPovwLBG2N$10lRE8Ul$IhJC5nr%Ub>Za4?KESeEsTmr3!{6pZJJ=ifXDX zH-wB`SFEe8 zA|eC+nL`^{&WIr|^i3LvypGjJKffq+aKoSWmb+l~Ugh@Oobx8r&;Iini}r{bFp$8| z8SQ%l7P-dwa3TWnBT?D)XSny~O1(pfR2QD!w3;)P0;4*e(XA6;5rA_?)4!c#j()Wk z{bjNI!+x7tY3o9lb~-F-VVYqcdA7|_2u7j}16d`uao%Keyn*pm9_*K=akk>`tL34= zaWG3js)F+V7GV;t(+I!1SK6JnYKsNFNoGW`w1Ppe*-h-5Z$3lTA06Eqs-4Ulc()|^ zRe6pqqPg)JlTDx3W^fic>+*tpKgxJO*5w*!JtCd;!5I>%pXi~;?N6Ik?2}hUXWeI| zEipS`RZh1ijyXJA`{cV+!Q+1#hksf9|LF}sesHgQ>`tF=5+4D{PWjqz zf_GbHD3GaO<_P*QXv*#>M_qZsm)-$FQ6_5=K7Z#5(0{SZkbEc{3XAf|{%<)nV6S02 zr^ztr)pvcC^(kt$>Dl}9LH$#=NFN(sb0~lJlREgi8{_-jRDHszr;Nbr9{SC$^eN!6g z2%``ft?HN*(dqjQWaR)fZ3HAOv#32R!{V4cM7Std*)Y)1c&TP3<9AiR!zqc2MM_7I zNg#~L?-<((_KIx~SN_K<{St_3>4x;^t7X&{)vak@zoP2yJuG{ZutsW`@TpXUx`Qxe zH5^7FU(s;QuxXP<46S7_0LgNUdr^k$%ae4dAYJPJd=N{Ll_T*GMi{e&I8x(Hl;3QT zp};#*%f61SY0cvo%c>-(U&+_Ipz?FwL*N{&JP z4d^>nl3xN<(m-$3Lo{|@1{W`2scSDbaAw`s5Jb&#A-TRSmJz@>< z)U=d8+!NxdnfrRL(;}61l8dAf)XUKh{~r;Vy5DoG40d*`dx?DI>I zO3dLusB}~=EK-SXb38TXE_&NZs+Qq09#Z3(u@Msya9XMeMrXCb4jf2+7g+Ps|~9TCGsA@PtpYwYCd z8`L<&&n1B=-CByLezZmZyw!#@6D|RE=~da2cO%VDFua(AuW8iUv1ae+yWGgp?hon- z?w>{CY!ZywR~}66`IKjAz4;k8R$BSijj#Lgn*Ce}d%NTMaj`q8t}`JU{pPo^AAx(0 zaz}^SvV_EdF3Q05nR#hj01$;b;~z}w`*s4K)eq-;9eqaw9nRejsaxwc0Yxk3iMP4t z1Ll}j>SJ+f%tm|SDRvKS1^A$%()k>`3Dxg9hg>*22P{*N2YlzMkD={>vdZAdf?-iA zC6I4-O_#v7u#~Km-+qU??Gqn0{Br2W`58I4vfA2Q*jUS8!;#RS*1hZY?z-}9mVWP4 zAN8A}S9pJK$NvlAV{$R>hT;kSdi&P=QwtTGFS#|=NhB0$qFUOs=oInQV=Xd3jfr%t z&N4V-ZQa<)WL_+TLzz&zETe{pEzoiB(T6IYh^`w@cMDLnum#m8@Y20t#&YX9D}xjk5+gEBN8zdjT?=NPy`HT3|!H>3o#V8MFF zX$cT;0NPGAXYGy#^%YprVV9OOP>KD9D+*Ku;PQ_q+;-4=m)XlSJL{L7L6u2hHzM@B zs@k)Fb`??xbXF!+?k2HWjRL3;gnIXcAG;%d*CjV;e8g=fl+UMh?vT*?fsQ@?+f|L< z=u~)Vo^Y54290IPFvEB!a})h)?vW_fho{>bCh=^H8Uvh=w?kVoA`P5=E%8?kOs3&7i@%Aak z&bVxa{ce{-WM%q|&8ww6r`4Kx37@?_7tr*rVNk@@#$&jH^SvvB?zYm;XqQ+hC0bs- ziH!dtvrD-6TPZznoWA;HxKfKF{u5sf)!LG?vxa)r+YNZSB!bQkJ|nCX;l$5p#YMCv z=`LKVA)4_1)KHZCF;z|zPSq(M1qnhRIvGKle&V@Yyg1CQM zufKEw)QbT%hJ4fJd4Di4k)_#Yw?Z4|{sa*nD|f@Oj|Ti|Rv{MP1h)stKW9QJw^DTL zuMUQM%KV{%nNF;$xyh9SxfM(A7Ws|*K)qi6*y)zG4ciJ7&nW3Mn{n14V{xfkA=%T_ z?R_k^(~@a%CrvtG`+CGZsy;5H{^agigcmOQp)QtBak6MlSv^;7mCT+RbAta)j)2n8 z^nXtpzj=_yXLA@mM1cCZpaA7{HY5MieYvXWbAT*YBFF$* zp4~}y!OJ?w%cj8K#x0Y(UE{%u-zwnp$)T)y*B2`lXz>qAo4ymO@%gQMfD{$jms+T@ z+mk~?;ZI8TmX@8Ll58RqkI0CaqXUx%a|{Ave-96h`6D{S(Wa|IQdBT>HOFx210>ap zY@^>oiHL}S*7i9Umw9Ed%YI#yWtAKE*FD;50>tbMt7`#q4+cf&;%^aR(qg91xZ#%J zHUpkaj`@>yb72VznPI`MFO(snTFTNuOtqD+_bjAawX@xlyD5HZgSkXVO+Sc<)fVqY z>H4{p@Y-LkiCdllm8vT4$ysZQyL8RqkZ?Ftx=>71Co){crmlxx>F9lJCMifMhIj0@ z&UP*7UuVNzppbV`88VH1%8+s~#vb>lhO+p3#w<7eUa$@OlHg>+=IUoESbOQNE~VBZ z0rU~U;Q`?Q12Ea8W`k5lNMumBWuHf(1(YCvjdG31^_^pAvg^}CN^u#0OZ@T`jB{$b zqui@jsG!+Z)l5ie=gk%U-b%fc_pPV=G?6Agzs;NGr`D3QQ}H z=cU@1SaL4bu(JVB_87m|B&JEa`O6OvtL_C=`_-prLGsmDk9aerxE(LILY#tlUa%G? zyPna}8S9N#WZ;2BgI&tIVW45YeUc-~5cp{>m8YxUZYS$VqEw|fSeTV|FP3hE*88p0 ziEDK3V(rSN-K*FIsU(==`SAi63E>z0>s9H z`@TV%Pn#!KiHk$^g^D~umY*W1vztRs1{x5V(Npgnn3S#Cp)2-zw|QqhMaXA|j1)ws zI^8%J!G`o@HPIm1Xx7)aPgNRlu*X)>lPbc#P+r^C(OJ_y2L$mpNh>)lZH&9*JbQ>C zjKy6gAK}JUsYR}M^$CFy;dHZs&_ONFnS0jHeWnh(zsY|3To&@w0Aol%T)r-V>EcCj zc0i}tix3%3hMs<}6#=gWq#cSaqD#GdCX=ptMLn(A-yBC9vIeH!=ID#KLN%PUmaisd z97~@wks+9%uoPfi0qE~R3z`unC4fWiEj$}Ir&_X(yKx@s1>U|;fwzIX-)<9n;0TEP zu9ts00wVkKtY+DaTOeBO=`Oc)v%f~ycC<4tw$_``H-gw#n^30*lt2Q0N-}gRQz!5B z`J15c0##1ZsS1W6{EI{AYl^T?QGI!=P!>LplKiU67<1`HSFnyeVSpn|u@6;Z?6w7N6|tqHah)?dD);PJNO7{fb>T(av2pDwpMWp8;-PlN)!&8- z3p2f)_dNaKI($Avn3(N-^{qw$iuZ*=m0i5ldwtOGD!i!)^QmIh+N7pcBl30uOWYVw$L z&hs0;+G@1OrVwk#wdv|9+8(_E3V)gn2LQSld(&4)1Fd4rDK*tw>~D%LLXht7*SBw= zU4WxbES5bs;Caa|Q=n~%a{I=2!UFQ#*9;!wu84Oit#ClAg|z*#n^&26MVuuUgt&;w za*FjgnN?j}54i*~&+PxD@=_fB18?Nj%Mz-At%dPg>)nZ=&dV{_jzV&2J5Y*#( z-sPLFVL%W|`!#lRaLMpKps#fE5AxJmpcm40x-x0LtTE}|22d?+E$b~Dl-~RtDJ8N$ zX$FQ9fL;o0Z_>+p=05?1kl@uC8gu1qPg^DqT#!&0uCg#2u{<#{#f{r>I005lE>shF#+K=lU=GbVs;%p9S!ROdkupz+~sAh#G0w>ooiGu zNvmODu5-sd9mT3d?F7c2TKZP_TbHZ*k$ie7nrF;U@qw_-yxUnM{0Any@P!~#a&^#N zH!)>?#V|aX%Se)hj&0MJE`f8LZ4!5vcquZjyD3>Sk#~NT3dwMLLs=R>y&`$?4SZ3x zeq`ED3$WDpBqk-8omTU!yTvdvwZ-=6E=R|Pw{uW`6@vWzrv3uB0ueYx;(6Z!(=Xh3 zU+1$ZxPLE7{j5ArmxxIR=^z!9*V3QDV&8$JGKNB$v0txm=cP4_ohXfny%~b_B^PK0 zn{K9MK~W7QGx(%*I6Q^tZSTtXz(=&3vt{!NgB@ZIi6v{W9K86d%{tS?xx!&u?NU7i+qj-6t5D8XG+aP zJRyp~Wa0gf&=8}uey1pI*#J##;`OruU8f`xpT)><=cOihuA!ZchWfD5A^~dhcfIii ziAQbLw3ol$t|;tY_m+8v7TB5lyX~RvvDd;5QRdD0;?Jl^$E)f30{;ZCMMqR^8Z)wF zca8rEolN80b^Ihv(YkAcSJWQPdHQkY-|RjN~t=hW$|P>L^iSB zq*?OSEYfV=-UJg`!DJSpBqzfB?N%U8xl)DD?V4`$j1iDYsV-8Tkwi$|X=zIJ%4M(^ z*{o|3kPzARa&r#df&Yq`GNHwX+d2!I6^e4hx=V~sv*ZmT}62$1`y z5gL9VJ7Ljg!`-h#Ge})N-mbd$R(sG(*4gW^lz_oVH1X8m`1%94d_- zs^CB3OHZPG*B&YortY1MNNHj(Pz=IHde5;b@>qruxkdo$3v&r&#wQl-(9jn)a!WzXkl$3E`&RPs0Csx zkW;+@77bt-EtVB?@B){oKeEq(cMV{RE7my*M#x+H=s_If~`tS$L07HJp&OP~-C6 z*zlJU=KnfIkN%A&e)EWhJaOakoxTlIu*d4d$A#f_50Ft!t!XmI*Ab(taSILaDput{q|?kiEpd&1dpa$W(0MQp+IG+2vnJrvVA%$^a|U9 zz;B#6)W<1QQfFk*b)vnnx1VDT=nNyWWZi#mJeEBWvg;xf*mi)$j!%pq$pH5pIyQSQ*f?#a=vKpOaug4NMP=f*+eWEM=O*bRu?;r%@>{{CUrTCt0MTs4=k`qkM(cVNe ze3?l)IV4o}=p>yz>i1xCLinsP-Q*DioHPf=C*jMmLYb%>jIKati6wG_tUK9(i>z>_ z#(=Z6c+b`~liWXGzl7q|laqP93ynIx)ayWHgqu{`z3cfax$<2v>xC2hwiYP>bq1P2 zG?%o<0do#D^&`6One{-X3AE%I9afQOI9HNA`JngD1!dMw!YGBNT=xMR3`^FrEhD4R z-5t#WpIP9`e{%ywa{$g#r(Ew=cIRsKKA`>&Yjjixq+Gr!Kw=f4BfO?YA4Le#Qzdi^ zcM%-sureykH&%FU4vL@wB`qkchZ@r1%mq;wheicN*#6-Cp+RaySwQ4yp)sW&OsYoo zu}eZBDlyetgcVV1D>V{r)!k(%Gsz9yWa;|aO3*vXejRMMO|g3 zmz%ZmCYp=eb%1dyP zJgx1D41t~ptlR;rb!Iw)y43)x?+joRNTtBE?JWkai{1TJK4~Bg?6u?GY-mPnCQ!}? zDtBjA+NRsJ5pzHc0?5ywuC$$l<}zv<-L8;7w~`gln9Z#Be&e3I@}8+yUmHoN4x>!k zHN;OCZ!t;ksG~M0@{a;P%n1;)q>#9Be{4$Quxvy?te56RpLVL!p_gx(JDm<`Q+%zB z*vznkfbU7wrVeVr&Fd}nLO}>w#G3I@yK&>(jj7GNb>q#g%9FDHLh<4mfbu8lYYQ0M zCb&)?cQ;0pNT}_9$|vW;#B1nx1SEOKl~8Png1JhhO@;7Z^EvXInFIMCQMHXfmT(8} zqWc_9wQIF{{aN%*Z+3)TQe%}wx3~r7Ft%Gw4iwmzrE-XCoq61+J@U2>&w+GFI`)^| zf-Jr#h{I~o{n~W*vM?B^!JWJv%W^<34X305k84#u(6a!lLa?ZobN6k)D|ei1pa6di z-~}`55!E3DcHe-sDl-+RmzXu50pgwbDxaNy(fvv;QyRa{b|~^o*34fPu2~^)BUSt^NbGF*6Xt{%ir2$l#L}8Wvt`fp_s%`vtAW zQc=}~7HZ$Rf9)IJ<=7Do8_%V_uEe7b&lVKCCF(Ait~))-G9|9c*5=Ayl?0z-2nBn@ z$r_Uru1>RjJZdK#OU@Nj_6P+_&4wWvnnuJPDuJm~6H>!pdFim(%E%*9bJ-Ue=4)V* zC4~spRe+?-%`{fQu~>nN@_RpB6Vb`oSr+?W(d{{vVRo)z&aX(y*~q|1!&!?$O=y}Q zF&)wQ#PGjNKN3dlY%lKtjJ<>6oP=hot?VU6pm^SE}s;NoISx(q!0n@ zf(k&lCAOw9+LrdSsq|V_z-LkB%JV0MDfHxQA2c(B6~2?dkK9Jf$K0A{Q{dLt%oMkT zp~2NJ=MOh!^RTYqEASdSM}eg9*0eknf}T35Tu^A}jBe?SVLxG^ zoz}OsvYk%1)!kUM?Ru9GoGmUXsm?C+?4}<0S5y|0@8gAecYkz?1TNT#PU=W~I%PV9 zTBzn*Jxb{bGpw|o87!5; z?xnD#AKo{REHf=_%Vi$dzO+{AS%y)#?j@}xDYUu6LEXFG>V$`M%tn87fl?|{qq-w} zYsAv0!ZEy7x2^Op^N45vC@JhOR=%!_Np4C=$qrAzV~U|^?4$j)UrNvTJ>gJW!IP+h zcZv}=6k^(0D(3dVJ%*JPtM4bGki)eurNyNb`H$KEyK?(Ln8Lsa^F^>r(8U`z?rK=Y zbujF^27BQS@whK@zGVvOBQvn7?zB5^L}1cyl?Wx81Z4OogE*uy1^X@mu2!z3oJ*uo zk^uqNUE>JjGpgi1Q!YucEG=!OogNBuO7c>6iRO5vWcZd|uoOS~V~ylMZe3ET$$qs| zABAjWh!g#*AK(^ts-5SP`8!7De0Idz6Jav(kKUV_R8_F+J%jU*N;+IrEPN{{y;9cU z$6w*+Et!$ce*6Pv?JLqq-o4%%3!8K1z)gllG{i?IqQDV^&AO_Wv0<(*-< z@r|d`Ad>{@RqU~{rd2Vq_X)MHJNQb(-Uo4xznRB@RKGucw+fdGA!iFFET&0>@DfiI ztugXXkgU(Sy3049PRM`OCTcL3J8?wnQc!#$Q~%jAC;PYw!sm@dq_jdTKPzs{w# zt6gPUvuG5zmu|z#4`wtkCZ|`HIce0{nd8|nB7+o^xm%3Twqgb=oalHdRjy!bB`^b{ zawe2(Le9JvB*h*S-P3nbHv)vP#?1PeX@IBu%u z&j*GIu2zgLQ`_J?$oztvY-yxQe{jWVVEl|R^Siko{e~C8IxK-<#Yay0N%cckCEZ08 z*+|B%9MY<{_zxNB4A+l0)f0H(a%FEb#jDg#dc0E+d_&vDrmphMADlke=&@RF+O$;J z)bXhsU{F;T*7hIw@9oYuLIi)0B|ZBGlHm6QB|6_mVpJMf7d!t1d-I{HO3mYr%uaI> zvV38+k(qPM2u9Tvb7aSg)||0v!SV6a!dB9(?ZN&*E9;DE7p9B4Hvaj{I5= z&NYr;66`IhXDp3uEqbRdtXeBo*?jtIWV&nblH+E&<7TGzlIM(iJ*jIip0#5yK`s}_ z4g_c#yK}I?)F4%}=9{G}QQuE%Zv+p4lJ?kE@A89I@9HyG3o#&RWpa1Dz6%7>2=dI5 z3sbw1%3}a`VBON<>qia@WC7ZV97_dRTIihP<(=o5FA)|1P4;d4T~kbC{u$|^`!8EA z<=dfzY{J2vVA(KRg9QyTzWRMp8O*?w4b`1Yah4eyI1-xr9j3EjZ*#3)uqlNV1N&-Jpcv`v zI(k+PM$WKOm>mn9?;PugzwXV(CdxQ#Jss@1Mz5lmf$QnvQhxAHQJ90X+2#f^v@bW$ z&(M9nqK~|r_YcxuVOcP7+eRlX;T}xwEe&KqEjk$Ked%-c108>2Q#e`{6r;VyX!w1k z->fY%q>Jn6Fc-$~Y|(*9Qz6P}U81y?is~>Hm)hJ_2T@p-WE^~+@);R54-BqK_MaCEyE|9*ISR7ikNT1EY>8Y+yP3T{C?a zw5->rfOi~dW6$jrmXOb3k^VpqiSu5`78_4ZJS#vn!!tyKj>2(r#)(M#^^2NQpRIL_ z_RKY+t;Xf^*p18{q{efA%ytSybU( zoS*khEG^J>eYz0cFdxmry@n^A2YN{jhD4wYZRq_1T6{L^d#(WH#;l4eAPTJ9InMGA!$C{Uny zad(@d#a)98(BM+s5~$!>oS;R5y9KAk-7RSG;u3jVY2QXI1!jvx%{8hS$ikyzf8vNZXk);|ZUPOa?!hntM`K+s+Y)l-K zg6pVeYhkD-5l)yN)$>c0sM__r%6L)(^wcLapb0$cb|KE%Ij$(w%Mx?0q}?PS$(?7M z;tdhzID^@m;1Xnk=v@=1L=29o}9J8qN4TJEL*4rYOsXq&v0>ulalCkK7b>=Lq5 z*sxG2{@V3x*!4@3oAEmi5$HM0bForhDQM)RStYh0OG4qi)gP-e!+3iqDx>)Qk^Pa5MsGEh*H)TdX9To9I9BX_j5y3WJlC#v8U#2W(uIXog!-KMEq#vTEfJ$O zUB@}s(yNdVps|g`qN$$1Zmak#ASJPujDcBEaYr-WgC0(=$1DDTN<<)Z!*P8L3FhXy zXsRuH-Z7Hp-Rl#XmCs=gcN|!cig|WCnL#c+8cfN!9s_51{sT{vRc>GAlGyrje{!=p4*OpR)m)FR@@B_M1(LDhA8XL z(K0Ll31{Z#Lyb2xW?L3Ugl)${>DiPW8AwInMB8p$5itqnC}iTBd^@cadpED{xcRF0ZwM<$tpIDTM6YtfCD^IQ9) z{;gkb`1gM`08JU9V;-mer{VEeCf`0J@^B;~4}Q1#W5hhS>&qrWef0%*W46OA9$#Y0 zw-w^C`d}-<9d`tUkG_qudp>OIx3}*vJW)GgX!6zI7k zbcT`3ReE^(MIKuAok76A_>ISuE$q-LEuMl3QZrjs=aJyNSzSqsM{KXOIQU%qig`s$ zadLU-`Ign5Qn19bxbk|C^$3^o@_&NNU$V1-L{m3>XLW~m*j+9cL`WKH$IpvyhFlU7 zGc858$NRTiG%nY#j3@Opn|9T{cFo{Zm8W$Tz(6rZ=gT8?Yq!9HY63OmJIDz}1fh>U zZZ)C8SV6eHn#DkT0#ordOim8oYWjfL8AZ13Fn{NanTwkj=P*@0z%s)y22Y<~afA3X z1WytbxOr^YTzKM;;rK<(I^m)2`NSMo)oDCS)aouYGx9s7%Ssz_)0G;K>np=B@SExyfC?F>(ObEc1e$pj*1qkCO*42$d|yuNr*|)qK+RIemylmpA=1bkd*)X)F<6FLD9EGEC$nDe0%O`QQjcr%C_B6BgBo9k z#_vcAXK;@ghE4><@qON+soM1LVGnH)N!F_sKLP6BERVEJZ@KXx8t2m8({fX@$i*ko z9wti4CzMWPKdCavGgGY+fFXJSnV9~*oMiP>DH1SXP>nc%rrz)mexhC(*5whsx=HWF ztaL^&+qAKrE>6uKQXgX6BYX}P6H`bsuA_l05C!2dP60_{`2LR38y6Jx>|T3Mw(<9a zuQLu7v*Gc)*eng}zUum>5rk|%8QGgiUnDC)8sj^KC(}d0pqJMNc_buMAtm3yaqsfu z6H0kzt@Fk#+y=dcH*uS)w!2x-X3ITrrs7g#x7kykW;K^JI;NjWEl`5!LgXpVN0yGJ z|0UP}JHh1*{oWrft8DD<~>O@fy_TFJS$$@&DmKg=erjsdO zt2`p_reJ_(ev30@qj9Zl+BE_#QH{#TPe5a+_|cmMKu|(K6*#k8X55kk2uE4LD5pcr zxEkH=h(E9u3OH~CQkAAbP*Au(K+^y`Onqr0sDIAVDhq%!ou~V#j%mp;^XV&)r_)o; zCNJPy!s(W?z~!&*VlXcz=lJ`9RUAg2T2ndSvxyP%cZWy&X9!TU0>Oq;m^9eNdl9pP zs;Xj*>^K;U(f`#7KI^J3u}a@?eBP|HvCa#=f!u3WU@k!uWuUDmxfgFW+}6kOhf`4* zH9-hy?kHfbk3I#pa5+(G7{08Lmc>b2`%5frcD1mGBh4h=);3ln`}vldPONx#paBL}_RfMUx+68E6-jL=lctyV3gv-DR zRcIQ9$WU<}gOP?YiSl>y(k^`H_M9ukPKA#~s-(jo=r>pRh|^EP#&-3de{E9yrIxGi z2--`b!{>4gfi@BK4-FMYh$pH|_$W_4Cyy!C+W2jEfI0b(DJil^$wZh3l=Uc zoVIb5V;#h_%QI6P6MuTQK@hScO5E?=vwugDb}>hP$}yvxY_W#Ih^xxkAFK*c`Bhf>_hZ{C{&v%PC&c<<2RYPLp{5{2m_ z-)2_ech;x(W7j^Ww4V=Bk$hYf%a@n1Z*%M$b27-^p`dj%P{P{K@qUk$gI4F!F+UdG z+R+AgN~hEhhDl+fK+@FI&3GQjqvKlip-v8!u)<#0_S*dW)K#Vpn7jQ1*((Oh(#6&{ zT@G=-RyRD1%t!vBdTdrVc)=YsixoihxUfX9y+{}IZoDejO({f3=n;9gWt@H<%wmn@ zvp=`{ugiGban&AWT9Xi$1ECzT>+r6b9&@Yk&K9&!tV5gTx`M4zD1CSnZcxsbjt<|Ln;S1epuG9)GD4jBaH%$2%ZN&HAtzS6~;`{Vh@>k4Qy% ze7vA=^58uno{Y4WdDV#Bh0>IRyV}R{^3YvdYZQl;;)xTtmI}Y=a;e)(vO!N5s}U&b z>@;Zx7^SW&7=QWuL7nAR2GZXy;DScpRKQ!IopQOST=Ol zRxT>(PbCV2MOE4ti8-3_aH_=qqZcR5t%<)UeNn_VP1ujyjdLm)K}HGWSws~_7O!Zh zLUG^gX;$q>@*H15(tCX#J)cmyZb=Pqo0HJ1JQnwK_s00(`&dma?k(H{nP};=q}0R8 z%UvKM9U+cK{EmUA?sJh)Gt$`U3B)6y4%jM>y}|;@fJ&<#_oiUrkWytm6Ms>ftP_}u zzluLr4Kt9M6jfUd(0I3|i;NrdG6UY>OO2C12um)9;5s+%>K^pDX%6fhI~+~|QHyk3 z2WhJ>pEPw4sF@~-D%=qNzA}nO;JTsJ-RFtku{|*>gs@*0{$`wx{O9ukQow(W7Xew- zt8^nDsj^rhELa!W5y}Tgw)noA_#-0O?CFz<Ge2i!Hi^JBWCtp$!1j&Jt9f?} z(6bmlT>>{rlrBmG3#!=8t?ch=YcRUlvF+uR=Bt5F40V9(w^xgD#oYK8T3_`X&jr?J zX+uJa^BH)%OMdqteRTYOH5>RbA2!5U16+d2Eey57BAxl+RCOX=2+jB|)xYxS!%wYj zJ0!k);!A+KY>7pGov`j3s|}7VfQ8C-+L@S8v!3OXS2Gz~DOsx-|7txF6mqXUteCJ- zrs`zF>Bey(De8z&$;*4uqVgtGT5juy1?XChH=$Ex;_n9}Be^lXwgSd415QUZM)xI< zvGWQ^i@}{b?K4rdUuyM5^Us{cBb{@0O64+W;|r$r@)@koTu%Pr?$_wM3=>NeRn^gd zZc5AOswBg_6ZgcKlFPA@wKGB}GR&RGyMtB0&@Ak;!JrWHBoSUe&ijc_skC5i(L)f@ z1T4?n&L%6ag?w`jb1sYGyO=T5>6Uz`^)STEs!UXC!<3T~>Hl3~{HXHqry;?+&tuzu z%PDhK2`C_K4^X8d^c$-#Pbx45eRejBeu9T|kAQLEmHcAQ)-_A^L!ipgHheE1B`_h=N08dMA8=!`uquN~?>Q*gBL`TKXU~4! z`WJv$*-W558~GWBKQuC}z+!GIRz_wc?)d@#Q z%}i0pZM`_pk}MoyT5*7pTPiuV2G2w99&Wh#M>Mr0i6)eHVhZpX&g9-bALm1i9q>k! zLw=N%U&k&h=HQX-Ky?xLWHvJ}Kz<1*4?vL!8#JILP~utZ;ThR_E!jSOCJJt1D^L0j zQ}Ku%j!Zd-l$NLa$#>Q}a~hxQv16iwIHzOjrLP)0(8Rm6xQ_4Q)|j56{~VdKu4R~A zQnaPn=mN{jCAxr>9QaG5%kgr=E5M0*88gC@mGd-?pH_9S)2Irqj>pHqVhAF3q~a+J zBjwVx99k3@Y7xEu75O4@+S_?sjIl}}hdrfS4JuhGC1xzfydgGTT`48YR5go7d{wzU zhD9$|)+pkH#*KY4+Z!FlpE8(g#mn#+#u2URu3GX2(`JW4e{1I@klmJmiXBp*>ZTv3 zPw@{7FpY3<$b#$k7eplEcH4RL*tO% z1hWHZe6y;A0bCZ9f-|;N$Z%|d$k4FD%_qzJ?Q@i|%FS7IbHlgH?j#u373!zK`5~er za1f?lRePRP%`;Lpjqw;27ji-sT4G(uHw_>D17EmE4~+=l(b;%H&Sz76>7zd+|iaSO#rz?Hk#?R#{@tP5I#jvYAry7YRG6ip-eoX5Dqo z`%{MD)|Au6Ami-~CQB|N{ zi7VlU)Rgw*q_&`!Q3*OP2I&h;u>xge_}|pehT;}WVOK@Ow+6ss9X&ks!VU(Xip(7D z+qjjgA)E}lEDAg&ShQ!p_?88q*zbthClyrhPI%f7$P(geqB0E5f#PY!kzME(+67!^ zEeW<-u~UR)a7S4OP_XKs@~#@Wh(#l+F;l%hm$f5W2bcnvD!t*t12e|Mc0*8rLO7z% zqGOK5MU_$Kq-HNMX&I&0%Z-=^Vecd5TMR~G-h3~CIuc-2hR*)V?NG>78Oj-LBz76M~D`+Y#^YDr&8QGP6w{>2JB?hj| z*N^$p-?o)> zWXm{XZla?%-myWmJp8(#iIuUf-;EsL7H@78Izso9qrSy=!fk{6bE+$z7qWTE1zDah z-2K*_!!R~voZKsz>R!!P5Qqrs9P3t(q76Vy=GIDTt=m}`!97g^jl>N!N=h|3O_9A9 zu+tw4P4&O%sozrTjon+M>5A=V0Q0`^(|=$Zz%~M924e3?qQn9MUU}X0MBv3I;0aA$ zIEj*4rURV5{%PeFlxK(ia>E&DrB@-WoZ>KCvd_Vc>fIkr;sAQ?_MtAjyzD&%9-+^> zX7)@|;2Z!hNG<*#iQ;I$x`b6W<4ObI zG*TdO_->>Fi6)XNWAU7BF$3K83wVQ}cJ>$OE&CeQEi+8V0=8Zz-Zd@j_YepI6U7|E zWT0Z+`}D@tIE9EgRSA7LnhZXYU8ivfwr-u;N$(=#8Dh4A5@Y*Cr>Besf$WI{s*@AP5z ze?JIz?S1nZ_SY?w*q1Qr+6`H3qTFiS5S@}leg-U@yvXfWrBiNU%Jy0%Z?&&1a$9DY z;(sz+PKVbju*ccWm6LGVosGbZks|D0J^6IO5z$-bO3mSCg~x4VrGV_Tl)m7-7OgSIh!4|fkuigul5d;{_5ZT#tvi{P-$ZP>59R5o}#c=G7Q z-h#!lq&w4!1fm*7uij8+rp4DY*{$8E3Ees829a7W`DRA&vxU^Q%`6x^L#qj)XH2N& zpq&^+;4dr4@a-K?xd9!+vuic231?sw2-td3=I4cpC6zv?tFC(;&WH0$niJr`tK2{c zc0XHP3?R2KPKqfqr)C0DuGaYK)QuYwNsNpik+cbZZP1>T1s@X5=R0S4d}S~DCu@ZI zxCwuk=g+9xm{#zU3rN9Bayx4JW*PoR?D;FoVsm)QRS0tEif)XqdKA*si1OQsy_}{J zJuLFnFo8ke#hCxQPdN~)|NJkPZp?q-g{MU)cO!S z7w$ML>>;Ga5y_NB&fYTO@`@fE4~QKY;7@Mu`FNnI4lVbhhYtbcm?}ww%l!)AtV7g5 zPCt#rla?(G?tvrhc&4qV0)T0Nb)X0Jh<(-QDe!G-N2PXG`}K%p45Q1l(!S)>1Kh$j z`8=@Q4(J@_G~5GpK`2NJ(|Zq3x67Zd0fyhKEtIVq(*mQ&-%Y;=eG;*yvsOvis8Ym% z%V-^xfelw&pY?t0P?m01SqP7iX~kQw2A`HUIs^M>o2@T?#N#Z#b2(V+I+~o>GAVX& zRhs8C-1Jx#70V=B6!OD&+L}a5xOd*F<)n>%+!`_ZXaDKZgCoQMS>MB;o4jO2*DxrL zEvfCGNW3g`JVMQfj9prrOF3TIbdkN&Dr;U_*_~7{!do#gY_sV(ZPbTVv++8k$bj&3 z9_5`f9gr{U05;RId|Y?5B;0o0u78N9-#bvWd*C^7;Z>$!PAxBW7am6^+lx+-HXLsA z==4$j$hX7U+dGXh1zGqw4h*Sp*+QqO?E&IwV5k(oqqRRRI4szZ3fx0(NK&jDY3sgS zH2EH`sJLAkNqlSQnj@%M-+;>0EpPQiO3QZa}A^DGri-&{Cec3munIJOS zG+*k9vk(a5N2ddQU^og`_6B%9jA&h*k6pjdhbl)ncBAc7L z0&ulG=wd5xP2z40Q$a}Ajws*YogcPDU>eCYQVqUOZkJ7GfWJ(f-7ftc1Z)lN0qO^F zp?Lt*8~gYGuQvcswb7kjVty4NFhJ^*x8*|p3tZ+idsTH=3Mg5Rt*b}skRFCNPk{YV zREbwzt;qMX_gwgs!z2@c5m%}RzJjP^TuF!psZ3$L+uyEpe|9*z%ke|oSMVp?fX(S! z;b83G`y4X6LK(Bdu8Do2_xqx3pLD%x>pMT@b)mf)(6FV(u51wB#hRZNm-I7C4Mx5mhSw3q-hX`sF8Ti|03MP{!~@<85BWA& z)lS^ZbYyZTYt7%kjI0TGpJ*8cDT0pQ4+P%NYvk{e(Bt6Kn{!Al_yOi)SneM$bY$2!zcS99p^bjd z^C8t=>Prb1-E|zGy!OSzOb%}ERf6t2 z&*Q+;?VDS>l@Q=`zrD-CltHS?nCX%mu*0q2;C_!6^+tH(`OzhY79oFrIJP?52cmOv zHCxz($CPP=r7Ct6Qx$jiycK*Z^f_Mp>xa=~3)RkgJ0n9g@P$7%7WI){Xzh=lp`2%V zqbm)NzaQjXmREVNuc<5(y#3(3-gvZZR94=d9o{Cxvusi2rD!{~3pp{W^TXMROs`M1 zO6TsS{Ffg9<`c&HuW21jz|BAV{6kUF=GD`#=HJVaS~VVAX8U@<)m!A?;Ow6?ETLlB zrMxgRnXGQZwBBr5LWD_sbzcX z+yKL~ngZP*+6X)8A8A^~R)o4~h1vk zi9I`z8JRRHMlZ#k8V~m31n2U&Gy|j8T`5>pTnK{WR9xF9hWYr%B5|?P@d0Pg_CmJl z&#()AnFer`SAVnQ>fEj-#o1=Kc$c>To`g?N#}?UX7D|;AOz9SqOc?z-w3!Svw1o?L zDHu~*M3-Ft+hw96xis~Db(9o8@^nhQFH9>Kq^n+KomkOB-2VdhrN9ivMEkNMP4j}D zcPuW9Lx4f?f=yF%-7Sn^N=Ev5L>UJT=R%)r5mLu^ubk~wCX`BzTr7I<;v(gwTC6P5 zObt@{O3^TrlXvsfBhLn?M5>o~_8P*bBycR2s7EShhzg(N1gB0^U$|vCa;=pAboJ(I zr*u+cIz64rb*s>NNt*pShpalHv1Yal9@IJ7HKucPqM3U!lXW7Yjn4U0E>%XA12^R0 zCw+b`;NncJyzFTMZk%a*h{nbmm(O$tl?UTv5bMma$=$@mo66z65!yvm2F4+|a>s|v zd9kDOsvc;rT-s4(jHD{xM@w;}uY~j6G4)1B*t&ejwC`yz5MD3(uZO2ETVNQ@gPi@w z^8yWA|4Ogz`IRXb;DC`>Ic$O#oq@aEM4++=cNv*Me*OD_Zq5zI;+=R7P`8mHhVLx{ zMuJknD-QTG0Cc`cJG0qhpRUK8DfHnIG;AMG3%cHLRF8yuZc1Z_^&n8Rr?*{Z0cPlr z&1%la-IVpH5_2}6gtH#3bu3|B_*{D$p?M@J*D!-?Dc9u|`y;df$N%(PZWR#G&9(ZQ@`r{~n*{o?~wFqCRGwDQYopPPJwXNqKwC^s_RdrjravV{-| z2|^OZXE93=#nWKZXJOMSYQ`tT#^;s*`-4kl3~@#eqy7z(e58)>`1`>;=&844oqlI2 z+?iq4x<>Ql?+0N8BI3?N-v_zrp64seAY?~16H<&r1TDt~nSaf|ENNUWW!T=kq=3}R zo}HdR%37V1r5{Sqi}qb7m*c4AgoEdmB2Xyh?kK&_89azu-LP{_U)n~mOCl#=5xqmV zk@Oq|$uaILZW9PvB_ughdvNEf%H?)k<2iG2ckAb;aiG9ISQ9(qK2&_@lZYZGOq_e` z`b@@<4ja=Ykj4%M&GD_q8VvxVcwLR;G+BEc?^U~|=~RbS&m9mc4cf`ZXn%gCv0Z_x zQ`Xliy>!xG%i+E|o!aDsmJoY_StM&+?|#|f46F?YPLdhW{om&lQ&PUq7~wH^ipKTn# zCptQzSj){_3Qgq@NuQGHHOnSa6j<9lx^$wVirE}s>f5z=uGoU8*h;IF&zoDHp*PrF z7q+TufU~j1xjD6F#!xyTwDm-#rlh5%-%3mW_|Z{`jfJsNsi+td=;Y($eG1Q`S(&wY zW0Oc`^C+H%9(b;R$6Sz;1H!Yk>IN`XsW|Vs@c)nQ_ebjg8Uwmd{XzbU^Jb528VB$$ z|9qpMAjr{)1+M3f3jJ(BKOO6>Y@|x2TiHTnEZ=r)7t%oe7}CO**i3wc1=m% zeVE0Dw^V;V_G`_!@d`Pz_0m_t+@f*^DyCMgr^(%9=S(-yBQaPa(vL-j&t&@*Yj#8L)z?qw5wN?%@zFEyBRG^X7hoErE*rg!cl*EFmHadFH+~lk|a_ zsPivkRePwlj?LG3#E+qUH1wqz_{CB3!)J-2a?^X%pu9 zt07|;by;I7PawNmve@Bg7qodQJ8%9r^qAgB3A9xcWXs`dEgdx!pLZ$F(0^g;*Q;(S z9(Fjx=Ix_LE%^5EefiMQO^Vb= zHn8)K1u_!lrB`%?Cyw5D)B$c~h?H=F$jrb$D|I!wW0Xld-j9{V;b++yiGGcdThQ{$ zFi178YHYgVI88nLcBZrriS(Y%ek{qE6`kKwRI0(Q3DWCtXd%$IYw|yFbP(+83v*gI z6CHOHR#BnNAr2O5P!m}9c(tmXo}W!L`99HJ#)>7KrodI(C`Bx5^Q$+z3JYboIc1H@ z-UKdeY=!Hv&E$n?An9;}@O1uM5?17Qb1r_tge0Fc{>t05RLI~Bw%sz1IMQlo(ns(0 zagsNcfnBCCfV3l;XF2=>=aEyWi~~9htvqqSw1?hEw7^WiW$Ib!QG5Wf{K|IzQ&-H` zwEpR$YwhseBZH+gy8Gse;jK4#KG84(SOa>;XNWl*k~;S=dt!Lk_rMZ^@~4-UZnpp} zi6g{N)(KzY{wUydB}JED4nmXf@OdRh$+kwf>_;UI^#nHDboao^yU!$mS(SS0Qv~ic zgb%Y`T1^uT_TAnZUI7a1@oAo!V1J?|a6JdaUfvvy_HYyfV84HGmW|8d-u^v_a)cf! zny+DN+%tk4-Q$q~KiN4D8ffyb=pR^mvb&CK-uq>bIkMHqIPEm-`Yr?T6d05l=9<8B z#wD|Bw7}!ajwqdNzZ zU3&1vqmzhl65}IW=MOgD(aUsnWvz=6vy@c|=%XWfc^C#5G_bdn`nFPR&D~XFY}}1H zpkD@3gK#Yp`&4`^Nc(D!k;Wtf@;TZ^x?L8r8JxBtZ3zR{JvA%lf=`{4|6x=n9+I4F z{l+?^w>OC9^~y^RMiv%^>Di#5AhTML8yV??!lyhP?5}~{SbfY2WUv?j#c4Vq?Eo@~ zLcw?s&orq#f%rhZrI0$8YZ!7PH$WlYqiUnfWJ@+F$ll2kqnbxl-Tf|}51wX0%z9)K z5aU*?_m(CYyk3h-Nd0QAT!pY#SLNdhYx#&&l`5ptWSSBLUK1!Amle-Nj#SkTZIA+= zmOC}s@ksf5h)(Cc{AZ4iwl;4vqrnpSse@frvm|iu$oJ^QWL#3yY9*^BOW<$;L~m=v zK*7b99RUC{w)fV2&yt*hxj4WB$A#6ZeAxQ}16nL55X}cxjt?<}J#Wil)As|zObsAx z0hSmZcP+g8yQ23%R>{T=16$Z=4zs5!<|;VfhvnEAj;pRS(l~&)gJtQC16Uop`?=(T zBcng(F9rxcS^I0uWC|Y`a=o_GsJgHLbt)>VSHnHu4qaFCM1d4;*6^ElohK8QeM%Zq zW7%7<`md9IV}t70PA!#L%UiX2LbhclL&~Fm{(IsEg3y0$eE3ILx~#UlO}3`2I zF-BUUFuxbZS9;{ThS5c}Z)V^uT1W5O*seaacHg`z^YHD)Q~JK3l2D{#vZ7+6u0ExV zHK6qee|{dpcX84OC!@<947@LRl9Po3l6V-2rx#^Mt9YvmrUHF5N>!GK6mr4~Rke#m zT#w?aC5rcVkB{oFqf4rO>eBAF++m(MVxlH zPyRd|UPKz9M^qQSUu0zJpZ4EnW7@P<<&iZjuFEd>MgK483@849KpP=uFWg;-P;Udo zh~Djyw#*9%o;2YHjS44WTvBXeoE?)Oz;6Zo&O0Q;GnFCBT7#K|mWPLjjxn#~jCym( z&N?~mOInHgt1O{M5!_ui>vX<`e;GxAI6I$sCBcHjyF~Z{1zBUw_p%9Rd$QMZsCdjt z-~yv6(yElworwm~L16xF-vqJNHe3ZicSf{Vd6XoRptPu1IaQ?_B7u5jwH8%onv|J~ zS`~Xf^>1wdYpjWj!;dd{Yc^j_dI?8+xhtCoQA}|SSlTI15uM6O*ZuuK3dL0&Dr2b9 z5GkqAfA=i^wdDr_Y30`@)fI;VD@5T39jmM>wnZ1-LOMovW5W$vlOsj70*g{L!m7Jzwbj;iqZ@gOd!9q$;px%U6ncmdzkg!!E!hO7?R1`I_*4O8tma@l-wF zBGHwRsVa}$9~nN3xg&umS$QJf0$ByrzG#I6R%-hPu0V0O0Q%_Cd_*l{Nw5yH{7W3K z+;`!cz|&v-UJgKFNbNN79m6nr>~0ln0M#bl&Unv~Z*&DF?w5v_i`~^BFefdti9k)l z3kY6sA#=`0KRqzZs)#sK*?_Zk&jzW9A^_75ee}-o{JCi;wHW{dQ5iu#hBLGW$Knpo zcV8wxk=g9+W2k-iOe-soU10L$!Wiz?Jbbr`$$*Wec>$JsgN-MczmedR$KeXp1+y2N zz=UK5(NwhI8x1rjPTc$E`0pSKRXNrS|TvYgnAQIo7Z6-H}?^CFGJ@M+ej%&@Vahw%HvjfoR; zCYI9gi4)3|ap#sc@YTKp%S^C_yYMhnh3d$sS=H}t!jB1jgVkC-AL;d9F6$&_4-D`? zsPt4PbpStIC%a2hp53NtZ+a#eR*pe^rrj) z;yLgvc?Q@H3{BaQ3H$dQM20jFRWE+(`d7qNklY)|zM9h^qhx4A< zvnwz)bggf}YQ>V+3Zv2y+lIJ{Di^HNH`>dD+v}HL<$3P>FTo!d*Dqf;Gwqo0Dn)?i zq+;_5$W@cF-t_T36_y{<-iwWeGjCf!@#>^iv|?us#KKfK-w(-6;+5$=o_NZ*HRSL} zT5zgR6L2{zS|}P;sXMD%oQJZtZchZVIKx#KRGH}$J7cSi-(5btp#JB~;{T+c|1*e! z0m9sL+gdJOj6^)K8>Ah=2>n{yz;)s69>&|QJS80}TbY7;0@WX>vSCE#;k={@W znKhT7ak;ete)l=SsQFI?=-m(Ka#Ne1n->nr1spa$nZ*@dnQfq5n*SMAVBxLzad?iR5Te98)hSR|3Y|+0AWV(%Z)rcqz>IZcE%s!STnH%FYPS3V7 zbC@PL^VYYU7q0#)s+jL?pr*Obear9<4Di1qEib_hnsuVtU8l}r)`nL_q*m(EQK7N7#}oiPB{DJ>811ydv$re)#= zo%8sL!R+jE3^&Ch{8R$Fmux<`-;RTPbbBTWQOOeXZLv%aOKPx^ao(3lRTAIYy}Zt8 zaxad9RXV4<^mv0@7?hQkQdP`ezS+T-5F3nLcyUK%OtjM{^F!=26Y*ag$l-@P3hF63 zrDa4@9Azd6u2J0^$<5ClOPrJz+3m{X%6cNA$e`v_m0MK7Wk!4wyB9D5AX?Xk#LhFEeBNx^**UKOSpcxvN%A+jUCi0=Q`bG8@Aw7WX3}&QiSH6@ zO*NdRL#TVQDkGy@rx8>96M!rYtlBt)J$?Z{gn}v^=CzGcYgm=R$iTY;jomW97r-xB zY$|C2)SSl|8D*3$P~Ceyu*p)3s3I_!COJT2q`2is?ykSTcb^-q!S>;8|7+vJIaNM` z02;=QxMEFBx4{Vj`R^UwBH)Wmr4}gbWcDMr-wovVW1ES4&cLVy?27YX7%h^p&2xWo zz}>6x2Q$5ctQFm+v-(#D7}7BYW+K5#7WjSK{n(vOHFnJpUfT-xfX+fV=t#9bbx3Iy zTqKq5DWxMR)Yex{4Y{uImt$A+-{>f?b*0pi&3Xp_BM~@eRMRB=3rfax)}4y=vxy_c z_1&hS@WmxbOPyNgINlDK z9BFfP_mF3JT%sDx*`)=9am7m3Qtw8fo7fD9oT$1?cUw~Mllm-~{ z0gM3B@({nH_P(>VZQtLOao4ml=Sgz{$lq|QIVrD}q0vM4(5Gg1ztzx_1n??j%JQz9 zTMY0{{4@GUo?6Z`u4_M0Kq+j_laPTqu$#T?FM49WtdM%g_fUpUTn@sN!Q(Z|!@dzR zGmi2N>awY}Ql@L-=0gSiJx{uQqrzIU;Q5M&sjBFBVAxlo9`o#8sfdaaTV99*=mWQ!_e^pi%< zH-S8p6GJu<_oCIrcz{VG&djKjmgvxz)N2iZxgL9#t!$D$5T*bDN9^8m>t!+~WYs6; z1Kc^lt>D#D&-_CKvjy9B@N2yWuwh063Ah6qHb=Pp=^A)`03V$g;E4jNQ=phVVed=m z7Y)BjNtNk8_W0;S>-*;IB?GOxDVD0&@Y7Y%2G|5cg5rQ`L^i?iLRc*DpvsWEA9qNeQW zzWe7>6N$h0&9Ds3hMa#pDs=PxV^kUJyhp~*pSPo6I9ULrRC^u4Al?0=1zfY^D3{BO zt78$`q1&$O##*c$jdbtNd@kz4U%1L#RFnSmZa~8I@Y5;ve=rWOu`VsX{gwAsBNiF| zimlt2J1_(g9;{^{!#_uT4hMfYR(Jkd`NTOSSwHcW<|_w4Sr$kJ{QC-=f15>HApDc7 z80_(uBm8r;DoLjBhl=g79TvYj+X}07)!$JnnB{qiibc6*w7P9QSqM+~^PP$79MQ|=l{y?fT!iwHIA5>)%3gK( zL|FEWPx}Na^5a*UZF`^88nr6RPhdI2i3j|dgCNoRbT*uZhO84uf{28$T_}HT!{Iwm zjgUf^dyQMsK9`7eqDAS;kuN2;C1{EKO0+2IRVycBj~OS^(_0Ds-F=XrAwqcN$Mm9E z51D&y4N8?gpUj=bp*zh%E{Vdq7WvMQL{3gl=Dkm}yWO<6F2sstw|BsJ5q2mF9GDs( z-*i#+r?huan*&M=6%6lAN#`|J6XP;S?1fQ?F9oU z6cwudhnmnTX+BaXq)(dyI7X^0|XO<3mPhI|~(Vj%jE7`$Tes;|P#LoGH25rml;V zmg=H?NP2W~r>va#xuCX1P?!0!{#>&FM{be!hU2YLCN#Yv+((UR!M&nP&em7vi14<%B!1aD4b>eskP(em-0Wv;WCU3wrhK38Wj;c<0*by z06Al^X(fXO3So8j#QsfE!9E#Uy zJV-0hD{O8PEhpvzNym*m=I~|IyVlv455ERmwD{LZ2x&WM^H2E;`3Wn-XFtbjp74ns zc53S|15x&Ezm15YNlp9rj@apJDdB_Cnd#l)mXjNN3iftstiZ^^x9*4@rvCZqTP3h? zBrxN#Y8BQgIvKZvdGnQ!{4GD!Ib{90_!2lQp&!7D7~V>wN@HgBEi3Ei&z0GL zF9Bu+`tRQ9rkazJlLd`WJUiOsi{~4_tNE<>EEbpsmXVQ>`gq^E+53ODZU98@|M%p7 zcsc)1`{A*33mu0tFqc!!;}PDT0@0%6Df9Mc6^R4cWKRydj|F$m8fQwO=sl=WFbuXL4Uz;@`4T27=uQF5)S}KdkrkTS@bXM!4NYf9+x*UR2G=kF8$Z2z2$kk-|k4C zIyFhNB%Ts>i4S>YW$F%NFLOkF;?G;|lyVtq zoJ}6L(``f;`ULe`-yi)0p<>Cill8%4e5C@mov?^TYVxDX;6}YZJSfnxFtY8p}Z0^6J*?wdi|ABMwfrYmrNCQEl^?GwHDbv`5BFfEN|mFfrZfqm&xeDYfPQfHo%Y>&`V!@wc*A6BW_1na(THviVV}e60?ZpZ#Z- z=zpFVRc%I0rFR8j(Cay6rQx)39gJ&}eszK#~WTZwKY6?uQJ>*by zNeWDu0*r{tgZ73lZhT8d|=dfXaI-ga)xvY9kauC8AI&72lXr!&~8 z9Q`qe?9iFmTr7vSEmP7{DF-q-nc)}9&cnVk`mkW00-=I;a_W$FsckQWa!ACY#?q}E zsq)6pHS6UJg4Kb;-XmvnZpwPY#LZZAc}&J*V1?3G&ID8azWURnAf#$=fC0X~lS z7Y5>pM5KOCt!k-A13s>NU{Bjuc+YU_I~+W>+s zT!NgCN*aTcqRx3+PDcQ~s-xwL~9O9XsVM$JwWNvP5>=5GRK>-?(Kov92f%8%s*n5nfW#ZxVmN`7M z)cgCTdy)gD8tVkVUB0=LJBv3Ws))82bb$5eslaE+vrFJfK`&%1Sm^Xru#Hx~OuEy( zoYR!E8k1u?Wi^(O-&}QVE`HMLZ;}$bss_2;bG+%?A=9-8y_SoZ+pzO#0)w~2e%Y`6 zN^lU>XE6|%^R<4(^}|NGEbC&&6n|sF2vjkqwiuQoYVx!ew4Q8G^^v6T|L5sGX!aiP;w9m_pH!Y~z-S8{gq5U`*KAm!17Ajz=)Tm3PCrQ76rU z^X25v!d4dOVfze6cT_NLPJ<-{C>UG9U>9W@os4o|jP2X*(|#0{5kEk$sbIwOdL!1G z?-5&>4o84>$t2sOR_r{W)s)IvmsP{SP#-MHtmxvB>ZlgEFJ2R`d&c}G=gQ(na{T|4 z4W#>P2**E8e11NajTTxFR(u(#RMkilnAF+9Sl)$$ob{wk7@JI^=x>Sv4p}S*vx1Pf zJ-JaRr0+>p3tKU3m*qwM3Rf}c2;N29yX&F{;U6CB;?5E5xSl8nKL@_!cE8e*gUaMs;i1CI*hb(XwM?WH|4 zwBlD>Ahhi(dSba}eJ5?2vx#LXUw8&9h4UCzxe5Bhki@+$PSIl*$yIwh1H2=Ei-EyZVcHQznZYAR{DroSRh}$)g<&r84+(kGM|Q~eAYx@o^l<}b6doLAe5uOW zDdvVu&XDh2!9`N>HVF786m=C z9ySo>a(S-6Tet;_uN37PJG3c#mygH;nOYHgt_v&kL_D&FQW>8vvt^P>%{Fu9Fo-xd^oqcL&aaRZNI8ckhR2Yt8XZfm`+7}EV87Wgj zKJS9wN-MqBT7O>c{SY$!HQ6cfM^)7Ori|7@xdA7CzYWk5!n6&2tUAg|S^;enk)%#D+Q|PKy7n$Q zW2OQKIpOl~2vmbrlX6QUD`r}u@|P#=4q^n-15i`A94JI;AYec!`EbeVeties1Z{2g z?harBbCHpbB-}0cFt7%uA-p5kqc%Qu9Vpu?8{rPAa8p>~>^S2Rh6{;l*l250bSIio zcD~{^a?=)TRSJ3b%z=iZW_A8kdV5W>eyu=_l-yfyvNA0t-h8M=XP0LcS}DPMda64! z>toO457i}NtXiKL$^5E5?`n^6FV5OJ0!^X)kM1>9%^hL=1o=rrueBZIlq`_+FZ$C9 zHq=}#3a>I0p0$PvJZ5+T4aVi+E5y-p4f#iH*yF1KO)@G@+q0g|56LSU_xy4#s@8%v z3jI&<%0D>Gqn&yDqrV>zZQyHNL`scK-F%uX<>+IzwN%hiq*Mw*)+aGk&%V51fOpu4 zt@2S`!l5|JBJQ@DiC6J(GMBsw6gBolplnMa@f%d}l^Ug2PH%A^C>)S}>F8Bb;VK%1 zR@R@9DKTDftm>x1gpfaPR)drrU88n*SlR9thpFJNu7$FGnzKlI4_jgJu_9e(5%05UTLt>wGD}^J4R{N= zk7;U0`_KFd>el+NRyp4GH&jI8M3-8U?Am@I8U4;_-QBw1348b6-MzbeKmJN-jQ2g~Igk9L!6hq7MKZ3(ZcxcgLrmT(l0L*jaP;99 zaltgXsKYYB3fP?s8Vr?+#)5Qq#DVlm;DfYK;l2Fy`zlT&@t3J^ate+|)+@?ffo;^9 zY|oS*$5^T?PZlUw`XO*3$)2x4PJL8xW*j0HQg@uTiQ2`icn>bjTOQ50E>p8*PT3bG zE^bs;o{l>6)E2n-@DwW7kcAU5OZKp*|K+Df5%LRR6O_yZ<8pVhFUayQOOLKc58VDL z>h|ZFRGBlvW?a}}%_81jQF|R$`2vlP+z(0M$w=v`RpSz1mWfTluA`IYOy0^8WviV< z?i+B(;dRrrpd)<)6&|17cDDZpa~v8&Vc_n&!ZCG30xfjnQ7RA0cw^?DAB>Q7E%ms@ zSVd(dGcz;KSy3Xa6R|o>iZa@dF7|A@p9O@%cXzUMjckV*_0P?TFBzqk>4cnva*blx zVqaEUWx* zflI#U%qf1Sh1}BU@%BQo#w(XKbXVV)p!s^4TB32wD|vpk1W%Nk58-LelN_!irPb(s z#loEC%mx>o8^JgVFsLCddCAY7eb(~TNjfMW$ntM~elDBLH1MGQ7H}(+-uqv%3-mb| z1BK5Qua9p>f4*+%*&Y!~1g9pOw7Wq!Taf*T-mRBo*shtjTYIRb#xhIf&cx7rRKk~- zuUKjW-M!Owc#e&7|4rxs>xGn%%2#oB5kshbktv^>%-w{R z-;%v_p=wngi(=~JkDCgFDZI3NAIxwa@`U2FEcMasf-T}FfeY;+T#<6D5c1@Gb1y*pLes@E>$z*tf9aisGhIP(F+u;*7BjRoHR~g7R(ha<=!` zxxk!WcM-rSU2tz^p?Hn64c1>t^U(f&U|hqaEuF~7AuOU@Soa8&(bnOy76Q8TT91UT z^Ur0v6$y&nsbHp9F>o6Uj_lCENSi#oFnwTOcJSc3>}M&-zT?Ov5+cZH<{GA@vW*^* z1+iZnNm6*9o&Y&B#?I*>it=*Wj>HZ<`IhVER3~5MhyM0Y;0;1;AoEWqJ^xiET>t4} z6tmP7uUT2t_Yzg^D*&aUS2TN&JWI~^UR6xPTKwJZmNuMQ3LnLNT+cTh!R{&aC)1(K zyd32_@CC7Z>xD&%ROEWMjfGaFfAC_QbGpIs|FK^7s>U}zcoPE8Vo{(w7WleAVZ8Za zR=$7!WBoFGT<=S3`z|g66~1RTMLl;a|7#k71EdDC@2A7L&Ng}Ai=Kgd%}r3o>v#t zmw)v-k(c9Jn%6^H%T&?UEv|@d*9E?mS1%&o7|$ zaWtoTBJk8QOA=bxO>Li>o>HZAek{v}Hss`{h4t?4i?_bT)4|7&@s zeFQCq5|ze>OaQZ4m9Yx4i27H%{~xD$|6EO8|7lmGE4FLBeDskbJr5{bAx!qSb(Ckq znkKJDtA6a>bAiJ8KMt!|)8I5sG++9W&~<&x9mVU&yG%I8Q<~9P@0(9|?A+oh$t$krmO@&4XqP>9<~T&fo~4T9aN zF{Uj%Yx-(jWd~5C7_oCQ#>1N9T*Ktbda4{LSk1-NZ?Z!ebh>XP$RX_h=h&yvkfPX!d z3hp&%6lid!HN{?7_Q#Vi*vB_ z?j?kg)MTL<@AX+AAe1&QnZGCJVylbH^458_vwPo_+Wn^K*6q2w8<(bCU!-JM!I6n){&|@YYf4eJ0oFPPxjt_gytDM3dpe-ie*z%p+1o;l2g~- zJMq#<)cmPu=(T?wb{1w$CB&7O+}*qMhQa{DH5_Zx(^as52je6dd{{mCF)o}_Sut5l znJ1Vh==9lH-~UKV{0+KFXe^EX>*V+khI?YpHIEmO8F=YfjX?g6TT-8Q)L{aNv#evi zWWRR(P<-t!?D}5&G)B9mKlhJZ6G*BiDcPI>y7v|;(~<@vI=a3{9sN~rdgzixqiJck zw)#|0_BZpLP;&L%a{~#n(}rH@k`8t)>v}QKMBMQ8GHIxEq-w?ju10W#0yGIfIdX*{ zZos({es2{XQl&#)wE88 z;%edyWTiW8hX}uP#zN{_eG3Wg=)#KG4P3-k4o)L=w|cSg+=nm<%kePI$|*YnVT#m8 zh7X|5kWk;$ixWJL1b7je%gW)`1l#O+f~6ZnvZ(d^mu**Oe!e}KHs9!Rs4dcWEaM?- z^?Z4(598BOEwteuVZsw;rp)=)D}nI{xQedeg$+G=NN}NhHc8~67u`~debpF@kB5KX zdH1Ek!wE>hBTk(e8)V+zWs(r1{hdRnVG08X?x4uml=hV4)bYq;ktqthqP!lJOg52y zht!1b`_PvAx39Wg{x+PgASYJb?@K)&n)zwgbKycov3lh5>-8 zogp&pa#x$8l!cD(MqaC(QK1dDe2dM5&B0n;flXd0PPuzpYEXqN=M#gTQ2euJ?jO4a z1>`3MSq1FNx@h4QOv056{&Yf*`VYS)<2ik6eOWSWr1`xM37?7DhOzl#QqBgSqZFY_(2X2XatNdz2?>)L~oO?3JP_DOMf7;nZyd6rV_d@C;>VbJn zt?{_DNRLHPat2pM%=h_#oFZ>4UN?wHe&DP9Eruo$Lf3?a;{uMLfuTQ*2%MeAv zk~93KHgEC4cxt@R3;vgkVMiIwrw`b>axM5z+~akHp20spZ_DbJw5qQ8g!sbZ z6ZGS5k-VE2rFFgSwIHAM9ojkH<%Nxfs?db*Rh@IzJhn~7jWNxRy6X|uJI~G~1&{xV z{hg{S0eF#$ksrOyjn4@;^fPq-o~>JvZ9 zjSRN>D0SZ8-HZ!8HKDu=xMXZJtxRwCi4U!uwkK{wz9w{ob6$ zT3JURKeY*SZ|!6LyXRdj&jV;q4DZ5Bdk1fQqs8d;H4E9Q&j2|DsR z4bl{r(GYKdAj?d!QB-JCC(jX4|KyxFkyvh|q(rUgto2@$FhexhiqFDtGPALGdeD z7wANTGO{Pts_MdJ&9kN)BS`4rNz|f4C!-<@x!i5#krriH0GcV%>&C?YwrFyI;zQZ| z<)?fk(b!^F+|TZBLgg(Yd<>B+b6+f5S7B%yK0dy|1I8IO3Pl1^%GFu6WldGR>I zEDE=U#aHn2kT!q!s4i&s)}z8G*)bh2_a3sB+Nc(n`>+*OC#1r6ro0?@y(Y##o?h$6 zN|&9MkFH#suQ;qSs>1v1Z}-lF!p185Qlef@3V8-5V;7|c=T)#0tP^4R7OqJhW2HPO z3gifC5T~1%a)M40E!_$X>u!|bW(~$~Ko6r039a%707&@RADzfrY5ON{V^#h%e-tg2 zY#~?+JjJW#xj}hM~TRzYp+z9Y#E$upWZnd_3Tk^5)%%L@Mj7AdiIOu5M65+&s!h@ekg9MkkVi_Cy`Bt** z#kP*Y#k2F&G`I0qk55nA>Hk4L|9_y=e;N2U`OW=nfd=+VlklbfeO@Meaa=AmycQn} zkjb=~G`jwo-C=~EpS&18GO}D(?|6|pV@I5r7dJibO`1>jiEB+7dcEr5_>bB&g}iC= z=lwm2ub!k9#J5nZOfstqri#HKdh)okg*yB~O@7NaQ)iO>Qi{Cu%z9f?tnH#$cl))k z*nrH2H%CQEdkBfUUtxVIE#lX@xZEs#-w;3`t@s>biFOb@#EuaKV75*^a(XH0vBfjL z;P(Nsc(B+ua_vG&a2-3^ve^vo=)4q>U((T$*BZF#d#&xfFDkmXU}XgQ3Vdo=YDwve zUUg>*&BA=T1xN0IPl))`Sa!M`xtR9XFel|AlREO<2SC@FJ7w4#IeSNuNrxuQC5q)7 zThu+~y~5>R6AzvD3%K$Py)v6yx)WkB)I`+EFBGwAG>`TI|D2oDk>F`Xa#C;xn54zS zJyqK%3kqNGXc=TSyy!l3(O+kqlrr80=EeEjcn`rjlm|5aXzX+;65Cg5wb zUNKr!wDh!MGD_Jvm}?SNe(G7~5XlTK>+LXMG>z!#jM>zzxPB*tt(2WPG4YAkA4ziJ z)k=|iqhyPDU1ji}??(%%EIeA%wJ(ehI-z=2JZf>q65c*h$$#*q*cda@=?Z})xIJ>! zPmP*=;*k}AF;o(HS8GDMWR&}@XiUP@{%Msvi$t)us#)^dmxPk)7@W?8-tLQEWq((A zBVY4^SmA4u7N-k^qLB8V z!`sbD4@C`iCez5~jEKw*cc;>#?g6(gDBQgC*bs4@iRb7+EOGETNr|q&<}(59HUUfO;_pe!Y^P+24-Po5~nF^UW1s@JhVNce{qg7@dOKR zH#Nb{#Kh;=1n8RqV!l4CrD`>Wz~BqaI`&4K^B{qt!l~J)hQm2ST{R)H5kc37ry2=7 ze}Yey@1_T-n{~p~LF~#9(a7vt;VPK(m|;}Vfc%9b-=lnCy=IPco7L5qA`_w~bhe+_ z?vZBIhDgP|bdUGIalME6Wrn`Y$kf%{6A4~#5`_c5 z_tQ(BE5~Okq7A(CMJ6uQm7ADK3oi37{}V9&Aw{|QFVW8b^<<=bQDz?E44s>cm`IBr zj&U<#O~ARS@jZvmW|xV-dn)$c-9$LLXR$QAj))!-|4p7JC()Txwm*Y~3&3Q|Ib>Cw zfDvbQHEK7Db)LsMARP^9KZV-+uzSZWeQekMzNT;X2~+f=Z0n-MqBO5P=GbK;}_4?8Q z_MJegwA95LN#Gj{jH70^rY&KcK7dgTq&BLwumP8o8k#?{{rny1(UK^_ef0BhIomcr zBGk=`C@9riD>DI>nfOyiO1*}hwT|lDA-`OP`74JPY^e2;ZlHm4R^C0qc^fw2Im-8J zOf8R#fWZqt<5o7(tA}FG`|KIseB)z}NZ`W_IL7yeitlY68_J4=`kP@fG2vP1pB!Ra z7ON;PWn1d~^VGiSsH(gQle?b_Scli_&F$G*2(0CCD~et=`}&9rK;@caFTlCv)T1q zLE3(|ZP-qNv8rN&<&YrerL?E$*CSR5LP88g#9Pas^+GDK1HC9(#?XOaElqJTTyYxqz0Xmu(RtSFq`cN%p^TgZU+TCp&x**&w-#F$3M2O$Tm!mYyWLGam3KV-Tp6U}FsE^R zvjLZvu#geV@oVu1p8hi~+gC=WL@T_G3|CybBT_2(h3hFn6@tO&%SN9wIsqU2k#Sxr zhi-pi$STQ7$CR4=*;$e1pGf6T(l%SP)DxYEc71_=}$SU=p_G9hZ*+Dn8MVriLed9RMo`)r&~2IV3fK>|voOfc=jcH(^=lLG-x=5wzm*&KCO4-z`iC^qd|? zgp6cBkNwZ&YMafK6wJG?Vm_IZ2z@PJ$$gb-`mj|nPPAJU!ZbX1lC8Z{z#gYSiZr^F zu`~pX5gbXew5+P1FL?$7-)nGVvrZ>$aEV8ulf{g&=P%|xrLx0c@3jiWn`fMks_%pz z^-al8MS5G!ZO+JQ#jrh=)%w81dd1e(iYfz){9q1cilES^I+RcQ|J^a=_b)NAe~VQl zZ_Og8Ot*w)H%7C%$wniMOxh94h^Dmen&?)pW8aZ4{;ikyf{cLO%>TBcCShwYf-{c|6MIXc>OcA~;bs&U@dg z)bC9l+4JTBje%WFbt%eTDl=#YJ2`KD)|1T4uWbbbB+?&V&rvTm1XaH@KIiJ;m|xw7 z!-GRcYP?28GD;gvXy2t$dKj$KqoNZo6E7LaFBa>7v6R_u4=aL1tD;duUlEl|8%q7+ zWLLT<9fXchwbr7XalAbtx8<^q(@ibJUmZNHsYCrH8g@l5^LN-g+^)JR+RHX- z+>V_4+d*mWR|yk{?=(+;ROa>JO}5ofXJ&s|@{i>PJvDTm zEsuvn)E;Ges|58D??*JjI;jffO;`Z)GC zH%}dGKb!kKBZ`i;oe5olm_=23w&b?5MKxdk*{d&?4Pl)UzCRh#Am1f(FX!fiZNc<0 zv1G~CErjnNo1Ti7q~+82S&7Z<+fX#kx$cZmO`ji0(yLvXDtzZ2R4>gcc^^hhPE1Qo zLVfdoiy+kdhh7LcdNEHF%Kqm_YkaKb=^YCA9hU_!%yr!&89?HYLm_8Df}`{^E_gkXP}@Q zVv}jwG-?bMia9d2BSC3hbfnfA5U54%?8h? zof@`&ZR`UawF@Z{mfeP03#(-(pu$_-U4?eX8Piq)$=Nm+_wD^UF2i;pN~PvP=0c`g zS(Tuj1TsLib`@cxm^O?+bFx@M<;8XEYq1!jV+T`)@dM50FQ48d8^~JL?r}lPsiE(t z2aoDCuPVm0^M;jO)aOLl3`UKsA(junAU514q%w+WMnq@|KWNmW>i+VR?druWXoh>JDd`&Qj;D;Mydr=aOd_ zP#VrlN!6TTWqttX{G*oGn_%&zV?VCI1W?ZJAgUlfqtdwd}uF@a;R*oO7L1v4SQX(RAS?y z0-XwXVQMzip%dh`=CG`g^pE6t;_PL4lcrJP!Q8@S*)T7%;PP>_9BoLmPlj+tC&pb$ zF;3X{pgQ~G3{}Swc07W{Vw{sccSCM`(9Vf34^I5Xp<#*2?7{en=t@;Gv$L&D&imIz z76G>k6nf?=l%>ya>gD_Sn)p~@c7vmn~Wc zj3cbx6@MR6-h*oIZ?4rWir)~wV|A0558Wo)&9+-0LVlC}+Wl@rIzxd@HVi2-S<|HH z5&hFhpq!f}@4>z}K8Uzgp@%LFUH^TK9sAdlrjSf(n9yEtEu5i z5-`E!lw~Tiy;1f4LHETNBf%&g*au}WL5q5(Cm0kZuP|czZ14S)3`2qkx2!`nS2pkMMMqI`WF zSc@;-UFg_$V4KGmr|E$b%Z#77kuG9-O9Xrn71@mMrz2}~(74$-?7F-VoSKvw+5+97 z?y^bP#V!ENm#(`G^>LkYEg;+gV_gv8z|-!1e3URT0$xqiwjh1ryDXtpr?}}>*(yfG zJifFueDC4+F9W1nYoWAJsCRA3YlzZlIZb{$rck!TF{|WByUWLHq$0cq^kK7tNl=#e zTG+jjR9=hhO1W5w{%HoTyre_WelFR$`p^zC)*gY}j*|9zV?UEwrq2_Fh3{DlHr&vD z9P_ma%BGcP!E&E$+@kX5fZ12@y_1K!6aMeszVo-{slZu7?JJ`Q>O~=WF&`nRp#YbA@W;t2mUmm8s=^2Qi}I@U-R=3o{(d{lpZHO3}qj%gNKsB^$cto#?>T{GC*mnexg5 zoLwrt!n=uEGUrn^T?*DjGfPg;89{SMty4x3Mk?lIzHwykbRqql8qv{3>0Vd9Y2;DF zAA8u*x~Ywyj7G5Bm@9_|DgN$bSns9MS;8)l>rM?k1x%ShSe;)k0maRi8OgevKqtel z4h4}U@#&ohGCGy8YU5+>!wqor89o?GJ31B+#!MPehD(S(Rb~5fIiRT0jctwt9&N+r zO@d8$WZd13iW=Cc>y>=w>R6#qqw*D(2j%6FR>dn5;)im`^iJS<*zT-;HLGu`=@tVr#FF74L) zWr?`w(bsV}_he+Sjl2$5Po91Pb<_O6hAGlHIT3 zGB3Nl{qx?xFSd%8si3Q*``Fy*CPMsMrfX|crD9B0-UzvpDb^-Tg_sT3DyPVUoOb;% z*Q26`f+_6P)~@E^e7TG2*jxv)QqA*DyvK{&_FyAkmQ9Y(iFY9mG_>^Ye4iH}pH@SY zXN4at+l+j=6)a;Vr~SfTLV=THktQ@lXZ)O$bia2=VPRz8PN>nH0pwt)@tc`H(=?u? zmth{&(qHsWM|V08EBN zt`xFl?aYY>(ONNTm+`)EqlZ0tCww19L)mt#GAF#2Ol0n8p;UZ7r+s-1(YvRbto+y{ zFQul^&U9?>X;ahiNfJ|Y{4#niQSSNT&q2-)Dy$X7tD<5`?4!G0bVOMyy%yp}4#xS* z7VE$c4CC&$Uz52}AU;(uEMqhnfA}_bg4!jp4ar`R6!cW$-Xcd(g^zq7UVt?^EMHSJ z44NzSIApcPutjWk9e1upj#7CYO*549U4=J2+owc2dV`QhG2pgK?+(rcd*!;*Xo$6^ zITH&`yjkXHCw?sJfJk_!H&hv(q>*WJnR8^_ywFg4{u+7r6FJdo$P1HD9b&53M@q>aogi6dGlv6Xu>AT!=ME@k2*^C0ZEt8=!Oo(2(4t zg|F6;LPy)RKH4&&Dl+B7#br2uCyKkj4PGtJ4k7#8PW6;=80G;{aM9OwmaJdpU!KS+qRFUL@kyYI;~Q;-gQ zuu?zAU}L8}te#VxElFo*Sll*M?`)WvSdAH*{(XCSfY!$iMg6dO6{%QA>h$tj=^Jhd zwBA=6JJVJrbJF6D6C+{;+O!u4DlL%lXdwE$}npU zI^r4CE~6|Gt4-ZNO~lk1!v3>GWMN@W<`_#LnX>tZ!3O*eBD>!Bw^9D(eg)qsGV6PF z+m#BNvlTG*^rGjn$}YVc{Z*ea3`X|9b?2{zW!!3XiOZTg?YsA|1~s;dD$g8xIUnYk z5hl>WGx{)b&fZH$kF+HFvR3~0dM(u@&bO}yqnPbe&8=mXIh0w0LZ{LLn`blXo15;V zShb8lcv0Bu6&(-faA1K9g>=UH+I|DReP%^EiDG--c!cYOz+BhR5OeqLKt=;;#fAn# zdlB7{5%%KiES8G3^7HbXMB4i)GHI>{&QIQ1{{cKmph_<4|ZcH-0O`3PV;EIfE$ z(bizdLS9}E2(C|0^zBV5j-5V`qE`=pGcx?;lMa`yU`$zhlUh6{t80=F4J_i1&(4GD znUTElmJ_qqO5KB&irVgBH9I#AtslkNe}$7RYUkG9_f7Dg)~@ZSu(A~3M@d>Ys*biE zPX>${SXa{8qha=FeS2a-+%u?s1>Txf7;pb)@MHVi0JaK#C>o|5t*02i|GPdAp@aBu zk%E6a?tZ&@uL9pwjqOQaGM1fi%XuD0>sCc(4Ra9wBnUhh5(sSBhu5eLOMSx^gyd_} ztRvJz3@D8xP1MASD|KU|oytxpp#xmLwwm8R?+)}l?VMMJB=rt1cYko>&{XNkislEi zsA8#UJV^wiI;XRTH?AtyGk&t{_;Y~4$^0HNB{_iQkyd9I8vZ0&oo%-)?@P8gr@mmU zYP!kWc&z0U-Wno$-gp9>M~BXiPFn?7d#f1QwGg{4%p9e+mJ6CMz8}$RwVSxWsWnLSh}4f4kZpRzzS$XK`ni^wzdi8 zkOx(%(y9bzKehTbb_}O(Fup3=T3SSXHN{N?6g<4!e^|ZTPaPl$q6C7HK+3j~SpZ z6d(`VOoP{j764qwI?Hyc^Ng+4n6xsG!pkFL(HOQnPRZ=Vzw(9#syvt#FLFj(sz^C_oK$P4 z`9C;&uAPje95_PW8^Wu1MP)*T!`p*hDA9uzzndNXCd>5NX4RfkRZ4c8gNF>2gp5*j zA8H@jiPx))l|eMfEjN$C*kzD?$Csf!1@^9hMdaGxmoywq?Upv(Mc@AZJS46j)!~;i z5Of~pdwP+xo1LV#$s(zo?N&i4(o1HamG#IvxmnKE(39WK za2mcpJAGUiGM&*l^{`F+Q`CymFo%l40)AEIZKKqMC#8Sgcq{*qv?D)5 zX2F7R_gcT=;Cw-zfE1e{(4kjkZE6;%n$PKO^hxgVy2}BCq5jx8kr7WV9&_dAmLE(WS@QcTLS8cuz`OdBM* zQ?J4Ko-1?4k(nyXSoTqx-2B7oOw83<>z??dyg_=-!UU7s@4h^dbhOC9buv<+P?0ak zArXULVdS8vmJGEs9l%0-ArQT+f%bnAz5ZWi>c0mY{#{ zbmg!4uG>4K<83wbdv+@mzqfwpnDah4#?*Kre2voaoBCxp`tc9u3Px`VqnZ<1uj&2H zbIevs%JI0%vM8o;@6KcdNst*hC7 z`c_{bup|NSXE~g2-NDWk)F{{*KfE+S&?B6s3dn|g!)q@C2xE{H@BL|ybua^wLLyKa z9wUTzXnzDdJS4?8J3D~N4{}LV<+VNn6b_Y6u%qqa+mVP`!a0l|o=wvuB`jY#DgI5I zWbFwc3B^`1k`$UtwHY;E-pt;ggJD^Ko}MvKNZ#)nnQG^0{R&36?CD!w$6KRr2Lu`8 zp~5*IShW>lXYCwR&SMUCuQULNxGs+S__yZ?R#sSwa$c!Pk<%$4g%xLg4wN)&LDUQb z&7fYOxpuMzg6YyA&&10UL?{5II8K0TbGI)bRcr^-Q6B(y58~K)KCMm*>=C;4hU2bF zqhs)pG`7&LPe7{Gflh=Fvi054aE-oy0x%}bFYPsObD&~$OuD*+Y9ds1#o%kpG;Qq@ zzD*N%onZZe6iLLD09(cO{)8L>;n`S%8u9T4*Cm_~hDX6C2#gMY+2;qfGsqWQ372?H zrm2e2SPsk_r0!N-j3b*tkwoRY?OP8F`6hvXjCWWybxKFQkvTYP40#vI3AYX0Xi5J) z9~bjYwXUZ@S<0_o{8p{Pti~7M-=b=r|1kWIW;)3LH{ynQZLwy%>Y5&v4ca^xO6X4- z-ORU@4L%O>iQS)xOSBT@{)D<69M@;C{pt&cn=`}%KALw%?XDNvz2^92ec5_FddGaU z?G)WmXw7&k?8SccYN78>b!~qazt56JxorLY)hJY~T?6^rTCp2lyBS+qKvZ@1Ya1p$ zYqruyf4Dv4ms1Jz6$T1!;waHRzM3ARLy!9Aoat|#Tsc$FltWewrLKOAsY|W*1ecHg ziMF>%M@^j3bb064F6#Mzrx*WgJ73K;Z~lAq-rr?2*YFVxo1T5$GRGs~E^D}Q~UpOKeS<2QyuiC<-;Te%;pZol?H&mxrH$9_rFQYm&C z`PmtW!}BSkW7%+`k8c3(?GNM8)}p2P&EILFIyPN4)&ls4)(jp-sI&K9YM3bZ&XuiI zd-u{GLX3rQduJp+F(7vt29_zeUop;l?E{stn>pn@RrnDZeEA$=2ixrD4*!!N!PAUe zS+9y9tPOoh&xda$aRaV3RupMy;*{OZ|HU}dwT z&Bbry2>^=7$?f6VR^)bZ5CIdRp;|uDK8pu}({-Xp>+2oJBBTb?Ixa<^543mZm5P+m zbGYKq0d-(egJ?sfPKCR&24yf>Y=&I&5O(ZdYa9`LA6>6UAUdPedoL5}t4ehOS2jlGG`h;5TA zd)=MdgTA_ws>v;xp}w7iA#CyJm=}Sly~{;k_on~w5Eb?6V7`(0Xc?TfySzkBjUZ

uc_;zW_4IS8;J155>$7nJvL@NT*V#YO>UwabV<(7r1@;Gwg z3EEm2$O_EPdma)lRLb-0oZE+e;@`s({%cb5FU4j5uI2owZAZm*E2)A} znbw}zlqkE-it9FWOXXs$cWdBpPvQhUQ`{CcIJ{gChEi!tEs4b1$}Cp5GFGX`d4hwy zG#?-DT^u#g43&!H$<0C=|Juc6*A1TXs1)8Zu8KYsL4SS~LH`fosR1io42NWO7p1}`vFiw*GZ zK;a2fIf9;3(b4Y4jQve;>{UAo01Hu|p$3x5!_*Y3(S65$a2%qghziPfB%G&wZi3cb z1()aI1X!U^WBMcJ{f}4f0ukdBdt2CY&%as?aINC~Oes(%KZAUL0kPYE>AvyT@r`=@ zoi?LOV6OpSRn^;Ax&At#Zo;8m9!j`xp(nn&Yn-5^ z2lgKqBe6Gc(=?(H8q~OrPNyXQ*{^OT7w_ls{(RnEpb7Q+KID@2Lrw=Pm$@oQ#$2d3m>YP$gqF z8wlC7QsZYox^n4HAQZd~?c8BGIw3g*k&;9Wu_f;d<=Vmh25caRmwXT@m%iyhtB3(~ z-+#QU|1k~Hc3Or+_`X*O;c_y_Ro-2wpZLp9FYO^Gt?nimGv*&Xa4;A(N*n90lZinhzY6)lzM+3WO)o;xe8|s3i{FN$X=mZYp zWLc+AAN;M%(FhRp>`TmGf!H5Bq3DKdcw_Et>GAR)DR9$3fp#> z480vUkzx3VeM*{_&H2qb@Gr$~?bUsveU6WbaE>MjRwJ_pD2aFDIW`(pD*nZ!aNJ zQV=G2kiciZt|Vh`!)ioXr`X1ab>a{97GC+xxa?!kR{SEcsqe%>%y>o+U3^KN9J0;53KO zc^I4?Zb>}60@01y!fK}`36@7>(o^=e%ZFj@os7VnO6J-uenbg)!TP1XJKftuB%laI z1k91Y2AyL33r8M^;1WWZ91}CUk!eh&EZ^2$`KjTI) zq*RV9#-enj z&d;sr(UtEf=1G6sDYl|}RKRCh8Mcy)Fj9rq?2w7Fr##XY`ONvSipjc5S~f21WA16D zAuEk;X66(d2P>diO+s$O5r-z5YNVkOcnbGf$B#{Ss>6)biHNB<_7Si#Fz$TciCZb1 z9sVVc;jzny;U}Jjy3KV&{BOs>-)E`0^U{>Vt$Y=4exFNFh$q*!<3qu~;8!S)8(ldM zl|yyUPVqyCgm9b~UL`JAhj)#qMkkC4TDPRS!GB4WL-eAX;mmi|Q_9hHMc3$2Rc`I0 z@a7O*1=-n64z>QF7kobrr-p>q9;ulaVCZ_G zsG+xTv5e;29DicucFDF6gHqLkuQ`us)(6k5w};7n_nhPIeKzDSW}R$RT{-)akgVnQ zfJMXg2Qk?~zQ^rfDRUpTRN9yxWT`fU@?ba~Df&=wR;in~m!2|<){ z&vy3Dh(>3Z11FFqYInP<7q_oY?On+B-P+J_aK43wT*4Z{g$Nn-L3iI`8qEhv#}Tm-T_|Ht-tWYb`vPK7~c3=z@U5f3#-g;|3L$Z zDX|^#Ax;x!(x3A(^lqwZk>wb%JXPpn`EAx~{7qcP{9I|Z>H|^37tfmLwjE_B&Z4C^ zq^U?l(ryNR$bmRO;!5At(M*l84=7%AI$w+7B74>dVQFYyx~tastLxm^ImH=dR4ro=^&g+`V zxY7oUd2;Zi;)Gk0<(3Pt3D@$KGT`5J#s1?%ng0XP^v}rxW6Dw3IG1Hg5eNS#_o#0M zHy^Uz#XKCU5weTSS>`84*`|jEXXOV8$-a0HA~>CyTv#Wq z{jPsBOA&*(d$Yfn7uQAWC#XCs;=yS^X+F@PQ&B{jWx5gkRK{x(2;8AizZb=xXTv!O zm#a+NURYPPe3=Kgdh(cq zQ3hI--)BP1UZE%LCIjt;oHFv777Go}k(=$L@6G`jcT6fQOXE)#WQGnIDZNC; zjw6?2sO%TJmR`#Ifw>+>V)+4IICsR!b|Ko#_*U|p^iZ3TM5=5|SJ8WY(Rb%xzo{8Q zyr}<-cID-zO@&tzcWXTFkchx6Z3 zUb`&glm2r;_;Y}$=xE7Yn7jk+_XAdy=lDpi@d9sB4Kp^^yWKDk8@B3Nx12-)xqcNE zY()EN1Ozkt_FWdqoAXsS?-x6@>PfdVL=o<(5cSL&%G`_Mo__1~+dEdbb{J6anl~Oa z(-7T$?S&XQ;9(EcNsfs4$jmmB&PR$xKYR7c)Fi(3z{0VmI=12Lhjyq|SobN-@cJBk z(;Pd;5YolIQEXr^L*;|0IA54)M-;T%Cq_EsYL5ONxAorui?@FMzmP8f^US|(b-w^O zH@f!~^P+Yz^6rQpL(^T?jOM8We&4uw-jvS}J} zN_Gk{9vO?BX&8xpj=d-E*sadCEOKVtOU5Tmi2^sIDCZTQA%y=xz{HxVw47xu8!B1* z2b}K+xqgCVXd=e!wx2ZXJ67#1pc|!U#D1xS`4<*@*T-?Yc~w%EP6Q78K7l9EV-y>= zr7Br|o#=x}oaL}7Xjs`|bESiLJC&>^DXH)5S?$=;P>~(%={hh9DF=edwBt;(HG~FH z$``}dRkyfzny@(1xvpeH%>hx$yGV7FqWyhC4^z}tTKsL3w7imMR)AXpNle~V7M?6S z$hS++ImX&)>*?v4D2>3QSUZW{MFh$In6q-;dfsO0%z40mTU!3`$bM6@wi+?yI~*3L zGK?>|__pkZT_Hndq5>}QKN@zy`4|dpwPB;^^O-&mP|oJ$)>jRVUIA=!@^+_&6pg08 z0iTi575A=jFMdn~s5?RX&8%h%B1TD|9ki6qAVVb(-N%lMD^fghC71eEphM95X$R+S zVIrK$0**!=fo&eQ^+PY*8N; zb9iAkntz><6^t3kXVcFeX90R=4lDg>YqtiB27p|g@S$Tz2gzQ05!~DOY}N4|o+R>z z01iq;e3meMvMDKNU%*aPPi)ZC+mbo~cIV za186Zw7)yrnBV;HOsiFVVlzR5IY)UYQ-jXVcdZG%Q7CVQ%CSBHQ=R@vT?`EEn#WMG zdJ}53BE!LSgGr!6*Sngs3Q$L>$DK-+9Do(l2W*v#YZu~<7@7AQ2B<^j`=rEncIEp| z*C{KUSQM7jucVD3r(KCj0UT&n9(74X*c1-PhZ3v1J^7spf%~P^K*3#3JVapSjRvlg zg4ROmJuou~SEgK0SEWYvoi$ zNtnEtvs6z1zL8Kl#gs0fADqyo>qrJC^n}=kaa@b<8q8jNRiJ+vV{Mm2;(^}UJIPXe7En$>a*rHo)cjGcv|)!tkoQ`Oa2P@AgQ%>Q zAOW)_E8(0RE~Wd-q(aw79#vZAfjMAxe3pBSU(iEKZJeR%*JjakPxnUnw)R*A5)6cf zcQ2{^?E(pA?dCN64!3*!MoMOW?3*JcnjK;wC@!sST#6t;yKkY~z2iCvZf}Sh42{*K zjafUOA?V`#ZMKE`MC0Hgu=KXLrx!+&E*q2MK@qI-M{k8anxsWHcZplgIdRU z22_Ga9i6;4J}Z#i7Ua%~kLYm$YB13^)+CFz_@om@H_1O&f$shc9TzL$tL*2?yC6?y zo9~yr5HQ^W59ni+Pu4bcHyLi9yyc8Q$9e00Qn! z;3yvqGoHFvu&IbO{0LHeb(F60UVsuvv3aEW;89G@W@YihmDkT2t!mvQ|618$@8!MH zUqlYO3U10%Ky3;^&47bdz0pw3DT)KOXe%A{8GJ$xC)7N*_2 zgVfS-sKp6>)XI?En~}BtQa$qsP?=3aUZMaUDOAc_D&*<$65`|vrn1=CKJ7*}4}tbP zaZC(S%?`ace7xF!FfyP~$-Ed}C`k~Jjhj19>VYu${t2-q@3u+j+6{ouLE+K(F|&rM zWQO@cI-!qlaLc@WDz(&n0?}Vzo?dhYX!-O(gETplkM#X`^j=+d`{8TXxOIypfxT6q z3M)O`WmlW;ELF!xmXoPxvJoo=a9Q5%^tNpbuRSeM&oyeS+Vx4T?%iGV=6+W+i z1eJ-Bta>o1v$w#*v{m~Sg+r}G0~teUEUDwu_QqaXD2dXb4Lnd^c|Pb!tIwUGvlTDS zNxnr~x=i-yamSs$+3ZiXGnzw`EI)2&Zjb*;`xiIu4`!_23JAWW+Vd^*(MX0zLX}w? zltE9pu6ZK)XWvtNJt9Iz-zFCaf4z)IOPXsS*xVj$i^2e z6X1*xc-g5XJFGC-&s2-5(uM<|jO$Y~ldC;REKiv@q7k#AS@fNfl?t}E5(@woQ zS3*B*g$<$i4>=izgplsp{N{XF+dFi6uVacE_srx$ech>j4^H6HIGnL2d|ejCV_L-q z&_v(v?15ZMY99*B`(V>zID=PxwP*FXStTe_{gF>?nCW=h0$lX5UfTEvZ9Hfn@j;%H zb$cz97wvZD&%g4z2Wp>UX!Nn!HfI8Cu_;n7Sc-7MrX{wVS$+uz#_r@0y_ zmEar`K0$S>lks(%kc`Y3pj$-})5mq!T?u1`Zw6Ojpr!g)35H?h$KxBH{*r20HLxLM zoDnhcQA2hmSNuJfsOw4_1%D<$4ek= zjCZ?PNN49wY|CnVhfKHG?f9c^7Nyr-L_VvUUoB}Y!57_7!fPtE1Thsc$?GOIucJK& z=JZdBJ+BD^BW(^7oi5ALe3<%)!KeSP8 z!KVgmG0=~2jX~N8>TB=)Rz4bfYsvX~Mdznev2FyWvRkaP^&-eGnkxM?8y)^(>+MT> z53Se*M}Y^EUb=a;Yj)^h{jqUryhsa|#hvIlyEe}N$J?2|+#YrZ@9A@LpDQB`0Q=8P z>JN(89$Mt?gobRqDlK$}N6;*pspw!ANJ0at6WI_xN*!5H!9LgHO-1hvsNef{4Vfmh zDt4CTOWmevWvt^tZ>*+jou?((z0Uj>d@(T#k1!%#WERJ_46nOWrY7NZ8`Xh4iww3y z0!s31%v|H2%`-lW8UDR2*rfeo-*jUCyTkh*_$dDAi}D;l@T;H4TjF2q_5b5}$v>;C zA2urfJsjf3}W3f7tVNG~=qm-gc92 zMaA`+DR-j>;;YvjxTCcDR`k4)NaER;A2PddaYYL5Wy*H)=B&(9jyHDgFloPgJ?Z%8 z=NZ$BLvfrg*IfIi=N}yAqSC|rN)BwwEhatgeAkVB_zkYq6dOJ@;X{GjWv+WAqHL{Y$IBaEJkAL#%jj;vh3crhU4}XL*Iu)g@vuNE8hw7aA*pOh@{Fy6l&m6Bz9f4zS+fpyjxFC^59c8N0NsN-+ELD zXw@;9%xg^_w1mc%HOq)!_OY*Y5G_9c=>57VhmtqOg=8<~PM8^*x5&3F60v4u5U&3F0D8px_fQc~p9wKA?- z#ltb=rzMf)`TEWp23a_VEZe*#v*Y99{2%@cF)?DxezB&@Hs|=WexD0^*kYQler96+ zd;JDO200(ERCnupj)jmk?Us<>;oxLB#KJ}`+Qa5{x6e}{Pk3(eU*hiFyV*z-njkGA zw5P|a)+N0zlsnWxemUd%Ju6~0=SSuUX*KTG>bRM zVxEtMJFFkq6A==cBQq(Mq%Ovdyu8S3`LCsCEBXRS+ zNZggWRBhP)(9vyDor)38q4>QF5nGS!e{|YT#D)DPbnX= zHMK4tetv0oyPK6#ENLW4U{R=n@y#z$4j(-SYbv>@*Z0`&nF`SPPNnniC0XpSz3c3doSe*q zFuyM}mOy6am;9Uy;-?!K)gQL964YmiXOo9l-yfV*Eu9g4Zz#h_rJsC#RP=Fp!>^MA z_x8y&?=mNOpSQHMY&7kDV5Kl4BEv?dA5y?`WLE}1{%>;xa0RWrf&wF#(%Ua=_poP> zsN=g={J7W`=xSkciJzJfPGh+GXlGtBzUdZP&k{qTipN^!^?bBfoei+C=wgj1VBR z!4rL@`$*oRTvWciw;q4+6yf^2Bgpal!d%i%yARg*sbo`zH3ex4QQcp@GLr*ISOZzB zpk;4|rvk3XHFx4S2W@R_uQ-T?a7M8qg*@r99UXALcd*K1&^EM8C$dbvYC!VH;lshF zZERj?gi2EBg~cU>g#kLUvcsOlZpW1=N0zNF=yp-ed(5)LdqXCoEd3DT>-N5V`#P^& zxxzudl1sia;&VTcWp8LVnPxL+o7H(bvwvcd?4ZEqAXZ*pep5|Njh|Iy85cETHpbex z{dZuUK;28K(y_3$71xhYk0iNxRMpn? z1&Vu#(?}NEvakt?ii)<8qyt=(%P+gTou0L|eMdUzU$z!+4?4g3+Bb#zp^$^#-sSuh zA6&oM?$*DCP5K=8tq)iDqxbuehq)>FY9N!$!%?7^XDAav_S1LU##x>`Ezd5s^O+`zO{LsLQWqZui#C6mc#UVYy18(nnI4 z*-7BHb^Q~EqMcPu=;6d6`1j4?TKjJnauk^s!N;kpgVp?a;&%_YC)f`;VA<6{an zJ*wsvOJ~~jC1@lENeknf7Fdb1=X8gR>2>A|?bP&iagU1E*I(EkiA9(W_?FJ}Vy{0j zw2#S-CwG4_*~~L?(q5ECqH2pqb!}L?_OoeLXO*V`+p# zh5g;k{Nh%!AB{R5G4M?)zcP3Q!;q0AC?>Yc(NPxZX!YvVnJN#LVza;Oj~^5ftkickd?OxwA2%kX1LY#{etjDeH>G(@t&Epv-Qy zmuXguh>lLWe%-9%c)QnqJ5-<#zS_HX@0MG?KK{my)v|JOE<5=owDetE6mTGbG4@(n z(Y?LZaHXK7#Xm?bDY3C!5rt9Nmg9Ymjg8UdN@J?4sioy*F-ghA>Y?xTT-4wl3I;wt z8`s<3mD#%W+Sjk{ckbMI^8EQ~4Gp0@!5q3<`|oc9bN}YQzrO1ta81+m_{igUK|w)t z8yiNSI5oKKgO^SvP|E~r$f3wWAubCg%e{MZ-8T=k*-lVNySe%K*Wt1c9_al1`SVFz zo1CU*YGI)~``4lRtlRbf(r-Wf8@r)?;WF`uchzQ;qlB`}c|8mIGF}ee-czjf_`MAA zQJ?#l!-BGv8ZCL6vX8Z0@f~{+w)y0n7v)YyxGKsmW1O3PSYm7y6Swsv1pcEH<%esvkKFJq;=6Y3$~gV*43D&K2Eq+M#j}3#%9c}~RZmaa`G0*C z=4|zEd@5jahlEI^mWoR1gvxsldiaU-wdel+a`-jY^=rJByIY+yY=iw-`rBV0@F``# z&7GqwP$3zHA3UWJM^)&R3iQhRcDK%kZ6))jw|#Q__~g=1<-|+s&+1%XqDds-@3v1htE#!X-qQ^YHwMjmJ-#5e!_172)^o4@ zn&$M^iq4IAJmi2-=ZllS19XzeMtNkTPgoPSNh5W6MC7rrugdLwo~}`MdC5*4PwyOc z`uzN=&Y!oZfVgDDGnPiNCI%9hXcCtOzCAqPwoR_Q~fEzfP1h*|y4~y|Vvs+l!nXYCLRIaq&_O&$=1m zM4g|cBO!|#Ob2u0L1c|tJ;o0CW~!Y^zq~z%z0CMNY+h6Ac~q7)Qf6e8l;Tzhto7>HGHQ>NSNFGd z6IQW%(!{T4vVA)%^Ih_)C^fJD4%EUg{N-by!qCrZT$sNwPlDA z*&akbRFJS2HNM_gvx(W^`|VB2*~$CS zZ{ONQl+}1XoQtZRtcgSVksI&fD)q}kC&y-!+{jGde9|@U!i9{2N~zdF0gtL{TPLf- zb9O|_Mrggd=K1g`vczr&2P(7U!2t^k?@0tLo6W@-gS`BFT`>~73W`+iV24J(T~p7^ zy-(ZPc96UatxIjyTzlm2_7T~aJ$yU8)J;fA>iQj&2hDPs$mKXW#QCng<<@`e?b+{j zA3uIXvwA&#m8MUtR&I+D+2qRnT}f*f7Qcs#-m0CH zRZv(#rO#-be3PuV_+`)0>0VT$kW~Z>!@zlZ5D2|@{2ca+_)Fzpv>M# zsXUGprn9s#?K}BaEKiWe7%50*qBqG1`8OvhBvkIu#6d4r3(fJbT`WcIleCFwl(kdY zuUNM1iGEKk)_=10yDkT>IIz<3^@ks%4S3dPN^bR+ac%!&d{YIbc54}n0QXK)Q(jGA z4;g+7rH-xNb&hvr>_R9}Xgv>v*W0T4eB2ou7dQ9$nkTtZR%%R2Kw$cE=V-BP^Hm%D zprD2B26KA6+shj(5_SK|kF+MP)6tQj(!Z_X=Rffxd;7=FpYLFGh^N$0Ii&FE_gv6i zgjNz6b@3M4q<&9z@Zw*!Q=d(={l7BE%*ng%uP;w`>Sf{Zm!FwFPLH1;i-_zWp9$Pg zX*8jwp~twHoW4AbAub>+{CMc!Z_UY?2^D5;F7CDN$&;NO+y2-A=Nxz#w1voh;(q=n zCeJr!b^h5#eYTp8P4vy{St-M3;pF4vlcwP&d~W`?0m)+NO_gV7Lm1nEN>?FdLWa1L zVjc%T3DGC$(k|{Ip}BG}zG{r$uBpH#aY>U|%*W<4X7SRp-}RQ0+f_cKzmSyi|IVQ^o7C?gIYx z2k(7c{4>=0{g0e$1R#m}VD0Je2O~t2$Robmo%|f(OZq(woQUI{*G{w_0bcdF0(2G-_XyR41#z>myDuq2Uz$_7{nUNd#@pe&$ z!7Scy&9;71Ki-~h05C@*O@&}Jx1i<{r$Z`{6Tgm6Ogu7z%>ROsrY>t%u!&;BPM1sA z{rz3tRo!#i*d!9Q`8$$}n)@r&zS~DS5iI0ZBe!$Maf2qe`_G0NHr}Ab#K&jy`5H>3 z5;*JT-h!2YoMf6r%%I1Ku@^Z(e+Eyxzba$(wqsWIwWEcS5MPmG*odx?WfJeXBxke{ z>+imK6}oEWe3y-0@AdC_4t4yUnV|F*B)du?M^$!?&R;gv?;=$XPO_KI?APDOd}A0d z-h#uHXvsyf{XIF@hWy~oONQYgQnXxOAb^-%}n%9 zBg*bz8N}t9RG&O~BC~E?bVh~*5~g$iElF^DjK^OPcAst*T&G!OS-RRgIDC*5S2bY! z>|g6t**6j9ORu~jx%CPPIcJyGw}Gm}J$j2ot|~*ZLp1C;b7o79%9brx6C}63yt#Ls zw%<5Cnop9%&AmgN@AS|)&F*~D9aaN>q8#oH-aC0Mk;AwvRKxS#LdZh!k{`qMJ|OD- z?+@qAjj7@ZNZ(VBIHk*QT2a7T_8&QCpeoutF2To%&n*DOM6@(!nc5CjfUik?>Gl6+ zZa+^`W3$*MBqWsh?d|I*ex}x|BJ0vC*+V;krZ$Po z7_foJFl|!ReQA5zP@)^q^2~6|I#nJv3%#PEqS@Jz#HXE|qkt;8H*T=%`%Lr*6FI7SbT;MT5M!Ox3`-%)Y-Nee`yN~L8#l!@9NG{Dx1FfZ^Ge-;t``2m}Qb@itpUHj7};8iCDJilWk(_ z^lIJ<%rNi;yW-;G7r7Wl61_w_iC|zb82UY#z@_I?4+q9ny$g)ncJ{Z-xHTe@$Om)X zS^wzAkFRA0C#R-PkLoR5#U{%nxMX#XwxMutb=DW;z0me4a78uA+X~_P839^qN$)MlNiset({u$i7JGjG9j~fkX-?;P>!YvgHP0Vc~>)Rtt3* zmDFO+AG+COqmMrKpLBMvL$7af_HLi@!Fv~8SDdIg`gEnDk`je?Z%~j(*5kr_n||!X_vvD$*oUtc_S4m-czeZRg9h%~8^D{p6KhXTDPA`GbsFIS5i>;5VP+3`q3wYflM1TmeF8_rD5NZe0AdYO* zyGBLDBOUBn!S$=^ELi!Fs@Lx4$VTc8jqED5UpLa*3GM{E!tlKT52wYyGXypsQ`oTK z#HVK|4}LUq2h<$~kHNTbf!^}~AAq$#aNlRmFoU@e9P%oRjZ=7Ur^)yMLqkKQCe`bl zCe04K;_8g0f8Uo36(Y6H3Ntq4OB`!|H zVaeoR?ew>*Axm@fP%WKq^*uxbu!tt+YrfRgo{F)tv4XJ8j3(EihugXf_OyCb$(eN! z?9arJrDu3sNB@zK_&#xmZ7C@!1gCLz(+Nupi|qg$-dYljbnGpVJ`qC%I_7Ir=V|&h zJ$E<{mzJv<801a6pj84iefZqWzTuG(%DRrxX*JXzfO_qRv%}nX`$!fC4uEV4YW(`T z0$g1_DuHNJnAREp+DzSZe=lZbbprf_YL&dLuC88#8htU)(K5se5T;FEdzifWT0Ej8 zDJdyjOAWXcDJtB19fDTZfA_lptx>2``aBiJl1nl0Mw%Qu(t1518Dr&@#5R$=w7OV z!k3%KBkVVw*3QUiYX>T;3VRdy_WHTs2LQsDe+dp_VN_2HB?`@!0V4@rqYYwedbG0u z_pk2xPQrV*LucCL-I-BI|H-#8T69Ml@~d&2kb;D(J>{Tip;EZ%OAO0&b#*=X`Ali` znl;oSPXw=B&{WIwtD89MhZ4ye$B@4tpnE5{HnfNWLPC*R!Pp5R(B8P+{_tm>674Jv z^w$gYSGZ%XpR%_0UsurKj2${ZpU9zE6|x}0iD>st(p1BU$bq&~=D>FQ!A~x;f+SSU z<|(%~cNMN)y{c?iP4;F4*w=QB9yE~Lcyk9}lcmCpI5}SHNt&8dHF|Aw?_P8BhBc{n zjjKE#YzMezysxKZeV}nAV_3^=EPG!#ja7J$3z7LHua89@3S;A88=5b znKs;KgrhnA@qAOKeZ_0CkqC)BU}38DKKdkiq&`NCH9n;H_Cw>gE9O5RIuEig)y zQa*4eB6I6$S-zD{qK^;X?w_alHQz`8Z$@`1J+fSuQV*yznwNT5cFmgQ-o9v!2v+97 z-|=FQ2@!=GfRY|eJzfP?=0&a%KTsJn5dQ$M{m{GrNM2D9m-oelsi~s89SvV!@q}x?T|A4viOYYMTK7 zH#3k3Nl1LJ9-P@UEb|RXF>4kCWl`a=wSF?7-Cgb3hedFLKM_y7apOww06>Xj;MpE5 z{vIH8UEF7F{qy&L=Uy9|owpCymA!UnI8D5(+j_h1w>UnJaim!^s=R4*y7b1Ct37v~ zKQ}RgzAVHrSa5untN{L()jOaqbpuhE9I*+YfP&ETC?Bdx)?4_%Pf(@_DvFA+c%QJc zO{mt+f!W#Fbx^B9ZI4>Ep}?r@H$D_G+akR*YxFo#I#KHZPIBcuxL)n`XAwMDrIUo%%!six) z2{iKq00p`-g;|c2J~GpPWSxRSjQ_Vc)P!3gcAsZ%O_dIwWu1!qiiC!{yPba}t%q4w zR;C97Loua3;NvAD=k9#oVS6Nr3*s8FK%1OY8p8VH;OU`b{kIMgoF$5Aw1|`xArYkM zh3F28tO%|TX-Cy~m_uA{t!es#fR|jby6>3zT9dR;tyiDhuZgKSvJ+ z9pJl(B;MrK#auoVJimVV^5u-IwEU}IkPvfmot3F=Go0_2sKGrqb_pDRp0@qy(bdEI zi}&9?4h|FyBO8_0ix$P^p1T+u2an8u8%{K3=$fE`P=k|+RCud%bii=E>diHoOgaU%lf^%~Ot3w&O44z`SFyQYR)7)R7+ z=7MgUbDOw^4Seh9v(@8}XFrkQ&2f6rPsCWEc~AbHtPWGnK{^7MX48U}JOHr#3qngl zL4mQO(qk~4fOn0JwB|%xs5I^G>O+=1T+?=~XT-(T9&{UA9)|7-6a}Oonc$zz0(*W` z)i3gvYS3(mMFpNH0DE_%pg@iOZ?Hm#xE0*g5Y=;G;K;?c0)ssVfz&;R$z(BU3(ST*~1?Z#f^y6 z1v z#HRyFp1>8`*;kTvX5WY3^%xW{?zTl+^hXs0ZUng=1<~eq7wshDwe39ma&J&{GWIWT?9xHkO``G%Atz3hRyhyc zTMr`IXrX9dv2_rVq@b{{>`t`!HeY4}>+_;SdrFEaY57l)q{u(OV5*&bqYlrBwS~o_ zYVQ#O!BaS|sk^Mqcahb1J10au^o{)dd^R=;5%4H!C6iZATz9~shjXA5aI)N~^$udT z`8L5fM~WO~%ywlXXZpG_w)3i)PWv=852M$<|0VxgSq-o(@zfw0A3~Vi6!^=Ti;Ig~ zU-Rk8K}JG?q5Z#sydHA6_{Fp|=Vhs~)xY1k?b^As;o0ZTtj?2FgHh18nx|gZoz)aM z7yeRSZ79GaF)<$cuyE1)Jy$Bly4ct2@{NT@f+M@u2CY`t!*T# z>*H78G}(JlnNVYnSbWMJR7vEr>`pQvs!-!ov4=+Ehm1 ztuZ25RDABpY;#K77RA!7ejl%(yb86*SRJ0E$r0Dq*5(Ax!=@J}s~KAjT{{6iUQ$+8 zVkqMK(zR=G?;Xu9KiUPAvVirB&0ApT^Ojsnf#Wv|aAa_Cv=?vDAeS7emnz9fB|ijC1go+rrAxs<3U_HV#onqpP=`S0^l?jfuX7XczJhOHB}{5DShOqR|dQkGBV5JVlP}cifV526c`}j zeHbS=RUND%9zG0`bBQbPH`1G<1I73ztHiqSk<%)|o1rHG8<)p|1n{W&vXu*4 z($ahE>?9!SLtuBR8bIw^h#vZmR?!^LzH_u`V&=~2?}v?zjpecU=x{(E31xLgJI?}H z<0yVaJM{Q?g@Z<^Z6zVn1_ArnSY`CpXqGWt?|4hx?HhG;AnsZb8%-SW;Ckt{S0m482mucW)q->zXyEgZ9~`9p0r2z zDX-?+P$6Qs{`;_3(qRTxN%5|(eI-}XH*MjG0|S%c*LT3C1&cEa#+owIscs6CrVpU- z1&st4cWpT7a&E0&w+^`41V5hB1NH`<7YRbrBCL|&uZ&n+4cb^DR*B{M`SB^^uK;Wl^}&l{uPfF; zyIxDBhD9}dRCWK4tZD|0u^SEvrTw>#f*ewYs)7}vj$;c(q3Ru*e0yI$uWV(LYBer< zRp-n`;S@`2Yk%>w%dTY>rnVE?ZIWHmp}ejvt_F>z<@KJ0>493V=KMm2gJ0^o^uHgR z;IHBhY-~i)CfzvEdHpS9x=O%i&}}a7hsVY#t%05{aybH%|~C|9CTlLdXZOGSId|!6axsqW>n&59rV1;G#i! z(g%s#8yl+EyL9Q2%`TXskmHU$y}ZWelBFxzOS9MBk{77tCMxN z`Tftf4McLVs0Faz0`2Myju;`6f*IunA7Y-HEW2q_(xY%rqN;&8eFdgdZ{Y_U)BoF0 zy>zL@w!mKvu0TxzNCahtcTtcF{(K)=Vr#X}6~_tI*LT>1ZS)T|}NKTGT>C0fPCK6Bbl`D?h z4fpNqWwGaI#d9Ij?z1_^FOQenmcFS|C&|3Zb4Zt9L%;)5G&bn!UPp&^3Elo>P;3Hl z((JpMzY9J&v2B6sI8=^cFbQ_uVj2fdDlr;G#$ipAgE>5Y$?BeMzh3Mh8g@pykeJw0bWrWkDZd~X`J&mT1o8wS{}2EJ z4FYOFbm!=fo)YMgFYC^pe(RxWeK{lJIjVLj&jn7Y0G3h>>y2n6c$ykw&p)z1lJMxGe1_SHYRmBz4GArT^t^jRU9$OUNKQYo z=0GuExSxRr6@|JV2{eUXUPilfrA-eu|Im$=;6ew*~;! z-A5__D6%-q!6Vx6GH>^nzS330&B)dSZOg4B239rS#StD`Jug8HqkaMI@t1kr=y2C0 z3yFwq@_eV$9J6Km`KAMRohir=hCPHs3y`cGB6+wam$xHvd`ea$5rUYLL?Zq3UY;kR z;hXRf`G!M;09-vhdydH02yL6E zkXy?OJ=)VXD!kb_jIt|rg&2*F$(B5#s-fohP`;o)MQZ6HT|`R-GNA9Xp;w$a@xCF9 zplkrMWfO)3_UMGAK#!Ep6BHIagyS0~r>eEA)NhY+A&3L5Of%YRMD%~eTj4{3+70}& zMy2}wGJ4ZR_@){oxe4uGA`8UaMF@2ELNF4*A!2W_ZOgK7IdkKghVzh;Q|jqw*iomdH2GShmMzE92w&WvE-oj8x#ZoOvVejl-ByNb40`Vaec)N zdo`pv6vQa=-<=`<1T3~ed?DDYs)2atSB*_ggt~*?z{qM3jwUYHs0bZAvJ#s)()?s4~Va~#sQe=Jma62K8wO+Y;_348%)4;nhUwTv#))@Rey6`%|^8?_N z=qt3KbBQtV`>e{t*Yv^5_wjkGzz+Gd*gm{HJ^dMI&RwRa2I?xi8?OKiTd@!RJ8gn%kvpX&K^TJW zy7qE)RT~%?O*e9HxOnwyj8+X{TRYY!T#}6hMD&r|<49IVe{SWdhj-u#8{I(-g+?a1 z9V_a66l8kf7*+f9#NErB^jQrL4OuPV7;Zf8$YAHGG;U*hrz5 zm3tfsIj0B;03rM*xTB9HYBA%*<&=~bK%N+vbcP&Y5#9?*tYoJ`@AV5Q^0T!Vh=1d^*I^vxC->1XqxgB1P@^o-B zdVG*q*T62f?W<-I#-en#l*(k z81%nly~B@H#y2|fbF`Ny-d1~)+z6ltz_@Mrhwz5O7lbl` z|9%07-islc%?nYFJTW9_T4(P8}_APdkYH-lj}NSNDqCRP|rY%Mr-wK z{r+Hws_5#Tmey7RsP@X|*-o$GsQ=)lW$UCSsqHIcRR=1Mv15^-uQ3Baa=Y`|ogf@u zxo>WM1PW~q;aGN_Pg2`n!*;?hxaqz9eH#Po zd}htCd7Q|G$6SPsW%;L{t2j8@S{w-gD-So^Crg%-ImpYC8R5gaV+IdlnkLK8;uGf%yCZ8vJ4 za1x|Ofw`-XI+um%^9d(qX2!Q4mlK4uglDQ4VZ7EXgF&v>2>4--h9^qv6{mENeqqIH zcffcj4da{c!N-5V=`Ab_JE@U80Ksr>qAqP=oM=LIb)~4m-xIb@>MKOf_Wu0TR(>9^ zz-jw|h3D^2KtwIDS37g&Oav3ounWjVa>19uCBR)m8TKMfHSZsVQ|iHwCz3eABWnJ? zck)HcamOM3$_bu>7n5-$URsX`=m@RLF#_ux)7K8KLt}CbRW_mqe#jC-g_M>~p5TGgruyax?)6uB3K!R{>GbA%wX3@OzVinA&Y?Ep^f>xbvI3SY)vIN_KFZT{Ep zQB~kpT7DT~EIAh12CyOQ3JV01B(|0X76W+qaJLb6(DD&Vw!PYyhW#8%`IY`oYS=zH zi5gst4uHsRQ%SwW`vt}^ZxXir{+wob)(t1M0|ySkYJtI_YB1&$y#uxW>vL3;l(@N# zM~8U+nJRpG?6`5$CJK!)v=up$z;z8cA1nNDzHQ?B-n7@hp53J_4CjRzyb834nxdDM z3uxUo$Y;MCp^nY0T=>B0tID8A#*0bDuh$SVYP+I!r=q;pE?dxA7+ol^;#JVL*89!P z_0bW5O^PUjpA_v2kU+8Og{4lZ3>NyK(Zr>YVBIV+4-P>K%8Sk2CfzggyEiFy-chd5 zP}=&uq_X?Yy~X}NwH~L3_PJFwJ$rd)qef{TQ^#2OHm&_&GaJ7s2U$Ih^|*c7+`{0S zBc;dIa%oSWsh8s4mbt@S!ZH20I2Vi%M&0T#8)gs0k9P05x$xIbF;yk+c)!Ycqw~v% zSHSzXj_3_X^E~Q%utiZE23^8cR5U%m2>v%5P6Wq&HMhF&9NSG)YuZb#=EsTYITs0b zms)beamw)-*z3+#X@QcW;eh4FyS+nZsgwMA^xAn3N0o~nNCzpEh#&xxoQt0&97l}W&3B@iqD9o%lmA3ZTmDT9dA~Thb^MwbkLA^? zyN(@S``bt==joAuwA=xY9dWUZ<4{simle>O84{6ga|mybb$Ld;hov4=Z(l2a1F9H_ z(h;;u(=uQ*)9Tzb@4E1iAziNT_oLZ>=qq9bym=&JdkNtctYU5ryb(@*XpZ&Oo+1^# zIGOLVmX(9c1?)L3Law4FeK=bB^ZT|9P8{JG?i#hHlS0}xv~JbJCTqTHxRP;U-yyF@ z%jpINXkp9`Kuxhuy?pt+qdSUmyl=KxEZV8<46nTv=eE|u#>ckdAw!JENAdn>=ejB< zBgOi^T*Y8th_1Hq7OYvIB~mJ#yO#Ajr}tU)tFISp0l^_d*Sh)lubY3?X{m?2t4KnX zEiGRs`V+sycv__LaMsaEgooobL91QCC%c+-yBA@!jlBibfE>NrE93|3Sx&yy=7-G+ z)!10=LjG-916oiByb2pD(6`M23=9-?sjuI~xzZf%77t-pqVVaUua;0-hNm~bw=~tN za0YlcADaf&l=)u@Qr^7z3XDF2Nrh}N@?eD5;!TR1Ha)GM)PDZPw(-%wsgW(S&y;Lp z@?A1ueJNDY(~~YnHleM-@{R}B>ro>Ab1~C0L%tvW8O_`DYmT(ez^T>pzORDyOQ1+A zI3mkxAbDi3%<4=n9QgK|T@W6+2y)KzlAF|!S+8ZB{3|`4>$8k9C_TbL4npY3HWz_A zY#hIQ`Lg*gUteD)-)iQ?hzRxw(y@8z5Q*>Oy%EK48fkV_LkatQ$9q=vB4IoDfwM1c zR&S?EdS#QNo5ILBMcP}hu_djgOzMbH$W+~X@wulnZ*(u^M!OT$d{DW8a%5HThUm5X8EeXB|l0gb= zIf!fP50ItiHHVtH%=8w0txlu8`|rhR&XSDnRLCkMO04nqK(1niOjXwIQMtF7fC+_@ zpa5H+0rOdyT;OZYzKA`DAZhd-tg^g+bcOZEX>IS21+@<<&?S2@$W;Q3R=kBip|8$` zjBm&=-@Dm~!+Ib7@xPax!3`=aDsr_nj}b44jioUzuH>=&{qC&IJ}}^x6F!uVbWP8M zaUCgsYhg*rct@sRLjsL3n93&4Zg$VUS4J-0$i}J&Q;gS@v^2(ekgl|~Fr;HgcWwLZ ziwp)Sg0x0x_Z54;l-*az9V=I^e4);pG8d#Wl3PC~Zm&3j zHaIO!)avu}T-nA${T3H;*f_Yl{3NNg7|$k-l8&rYBP>nAZ3&oa4wTy4Jw*mND8bg#PxS+;bRL;`u$3j$)xPGP(F6^ zov=7EE>cL@7xY8DKL44wz)jg4zz@|}B}v&Ap(&AbXmJdEA={&?KRo~bCwC(Zq$$gn zu?h;j2xH-bC6+`{{MAgia!_}rYfT5}6cWp$1jm#7+H5vfLCoQVlXDu;92b>oc$!rg zm6Y7LcTYG4M47{}-ripMd~41H`R0ikS$bhv-i>Snqule1?mg@5Qc_cMYhVN}c|JOw zlV&UVgGok?Ayaqs$G8qj6?f<6#yAhAd+md7c6@kvxNIW^n;ai#n@7^%`lCf$9-HoH z8fVt{u{Fuq2?+}`Yi4Hy{&y`!;#zn$IcN2~J<=sA_dJg@UIn?}2UbRDIC-wRz=dfs1NY+OBMsLIp$8a%oMMv~P% z-ij)nsvVSnT+HSni17kg`?YR86qi$2(bna+(z7mw*qGNZK-s<`<92e?-TQ z6MA{FmS0Yd()L3A^3re3#z6=_7KZ?utWW=Vzujsj4@aAC?cm`k8Zb0PR`3fS=hKl^ zC!rWQqm{ik1wMEHxfb5~@8X#I`tRbL&n$N0ziAOgH|8hhicXzq%T%`p+hhE;=xjq#d}%b^2MaulL7iLlm{)!+sN_x^C&aJP(UFEKeGaXman7aRnb_+qv^-j z36_HjjUXvSZGVPwj)4OHTV1&;By^tKyz%;*`nhq1^o36wvR!OUfomuFZyV`@Z7i!n zf0&cz$X7lkw~h;-ze(Lx-E;@TYq^(-gh& zH#VKz?Q4PLN!AnJ>aR?w z55wd`MtgSZb^->DB}+!~(@_9UX=m>G|J2>-h%XTh&4|&IWaXGqi5?=0x4O zj{=)L-(4UZ4NuR_!Ur2OdjLe3%iC%->ngtqYb&Sp<0`mC>3o;Z%rqEb9Sz_*C;F9O;4 z@b9=%#H=T31u?eAXqtf-Qj97M$zHNIE&o0qZ7%V_g9nQZtb(|!SNBe?vi5ztCRK({ z6C8=Ft1EeU=?v6k)FDO|imErJo}B!!e7Q9>;w@iK-?z(Ki0=4sFDDIBp1lD zRROMM`td4HndW*tM*{LVm9HNTWZ5Z>dA$-z`ui&#o_T6-Ci8*~lR#q0b3b_8kzqXwDZ*d;8HRR}n z7+BbJqJv(j01Lawks!*H8}vyzOkou(DJfx^R|IXxz4#^Osp@ul+|wOQ%9hBhsa+z- z%;0aZa~=hAN~%PNd%+V)V3OHN_-L5t3A#9aZPCrr(!u8;U2_|t)vBOoJbm`8v51%n zX}%4Mm=A1`BTzxd13q6pLF~YAzq;uNEDJk`nq#;<66@ZA$^U)&(I@2Q zO9)ArIElfO#&lq!({H^w|07!C7z}$1qZQgC)}Rks35bc^4?FT=u98>Yxg}P}ruSIt zAw!t1ho8XCQ2p)Rfc%CHMmBLS>BJpCHd;P4K$zw+Q=(`CmqhHFh7Ds2G*i=d|Lm15 zpkE}45&y)*E@S8(va4aDR6Oh>Hm&HjFomEmF$hs6UeUV5ChWW-4#Vl6+}iaRe-dYT zxI^vWJ@=fF5}smlIL3#^9?pGUMOfIdMY0QZFz*v%t&ChWV@!-(7QP25g#()s`Wq}%SX=ZiVs^!!pzMT-kA=|uc1O<;ihKRK)Y@|xY%SYAS7 z%&^5>5CbZhrbUD>Jq!~hc0rLTvZ?l>=DRRu_R#6ghQZ+-p7^e#;P(dw?>cwMmP``P z(w;-e%c)0#w}5wJ?9B!DPuNn3X>vwQ9O!Sz{|0Ngg_ZR!hQ7Y5tGzOmeag4Z0B;sHB@q$!us39scId{Kdg0T3&u z>?IXns-IkLl!Hr|n?qeky=Z#odx#h)rAcJz%QQe-Ya(p_5Y#EW#rN)A!LIp0I{$CZ zju_c{3;$c;N4!rC?OWmPEnJzos;1Mc8sg3qrK`jP<5vF(YDcj5|0nV@qRQdG!O%z} zEPLqI7$cfcX^{afo℞Smim04*5Vy{4f>ZD#_PspzkJ#FJP&L$rv(1YY|Tq%Ec98 zqS<=^c%ATeBH0o1e=r?1$DqwKxwbi8&FS%bUv0{|wHr5HCI-Nyr4to_hxh1AvBy4p z-q{%vovS)43A#c-KSacdPjetREqx zmZJv;`Z$US4Tived>aOgj)1ZSI7|eA*c`wi#IJVHKq4}J2xYMR-SWuoxU?Z)?i zFoS_9rk*L?CX0#17$ZO%AsLuo%6G{^+j&~5zUjRvtWl3)@}*-}d+{&HMw(cw1 zr$p2&R8EW-<>yDiQK^PoM`qoE-C}@fIROU$dnRNL^-Hs>3+_E|U>9U4t;Jv6Xb!K`uX$cpGvT;YMtElUgEz(fr9s+ z(?Ju)72UsMas&Y6RU(thka3_H&_T+>S0fqnw>xX+o;|Hx0K8ct|F%F=JBA`mRP{5r zYQ`07PJd5u<>KZ(0LMJne1_9euL}}<(ITSp=S{j-!>l5;EzsT%K?s7hf)MSV!a+2P zY3?)(YZB%eSX8hWmoRjsr2t$cCl`*?_yeZjGQ65F&8*uGG>)m7by(Zp2Pgyh3XGMJ z+rCZ(*b&e3#G~pqNP`yrO@iu}{XqD}-I}l&yhz;<0NC{8*)v*(UB1g5m+rNY*P97n zhD$-FAz`qBOib)PDA$HSiy!OuByU&9E-fG?hja*dAOGfcYd;775P*!Y}94#DQ;(@PPA!fX3X5`e)0Rusak232LRg+&-i63&rf*c)0@d8{e|rW{dYB8z+vWu<3jEuBcX6t9h( zd89#7ejKweguYssjYfvRHVZ>bi_)^{n1mo64_*Fj2MtlyI&QVj+?j9^(bR zOax!HqGHlbe0>g+DT{4`TYnwCSq8_e!l{*yFp-lEVQdJsA78Pd2tSCpA`+&w1?T2* z{r;Kf_NrqX67tS~Dui@_!5ZdyB+?%Uz5fR0oqq=C;L9hjph^9?J3Sxs-ib{gWZ>LD z=7o}gFjzQv^5pu;%F6$zr7I7Maee=7DwJjvlD3%~I?B9aPeQBq2q zF|??hv?x+ZL}dvP#YE9|GKG}1ku6CmTm3$_@9z)Sb*{@X^UnJ|&vW1RXM1s*x&v>) z%7s8EMFcZ+z6H6Mivp1Ef$;?S=oS#Dd zHs1erEeqZJy_Fd6pr1%enuuS7GOMYtkNIN}UCXm7C5 z2#tQULi9^u{P>)fh%-gZMG0rYeVCk9J;V%3?j!iQ_h13~p>xhU59@8m*I@hg$$6gR z&uL{ELvc9COnO`hf_DbK@6W&IeSMGuDijr?80@P=a07SNSi9EIaS&Dgw4sP&ij0fo zro{Szw5WSHjY;iePol4VA1+lPCo2B`1tz%1=hGMN)kDQM+m@bJ*TsFKDgzg&%p&cb z@0!j@MVUMJ3FYS-pwq{%`r`+L6u@#VPDgOxYWGVKI4BVxRqqSVchD>So4(R<@QtWJ z=Se(^RRn$YMbgfp9y@%4Q%W-<_SCz@^Mr31e)(ucbV?`W$%7R*xw*o2; zt!VDt=T|+2=%-hud8U_rdAwZK-8W-T{2EQhu!OdiD{2a5>*T+N<8j-s;Q@RRsb=!* zGMAw}(%Fdih=^71a?kYTTsz~cb5Q#O%KAr}T0Nc>39z0z`O8}fj)95;kHLeQhQ5i| z`R10L*Z#qj6c&H1eyoA5zw{9jL!o1|wj$XZIuJ?xyKb0jw$qH^jq9M1p&5CIQ!~Ue z711Y@OJ2*mL7M`D7l8Y&xcTjftYflxLQ-Ooh!=pJ0PD>JbKOsiXI6JeLE1jtNp3kjn)fr<>GV?iuV*Y*RW zKVMS6s@7|C_8`KfQuDqKqiMR5ZzKJ6HjBlQxgjIq`Qng7B_s34JC*s~wb0Jdh!3^B zEGz54=m=F;G+2N~;4&cY-qdFr-#4bUuaJI^(OJ>kYb`r>YlV>;hJd(maMW?6V8cwt zihf7(Kk7wcy6Ex&G}((?V)jcLQHmEaDk*$jaaJ6cWys(`p1lt;X^flVV1hf~qwR?` zsyCBeD-UHbHRDjVax*ZzL%e}$zfvJ`@u~hvD&gjm80$?te7HBox3kgyQiF*@T2m7M z_pG9_Bp@S5^I%x~MYsV(A+teth4n=#EdC>*J5>v0X^n@Nv^;KX`#I0&+La1`GswLH z(%jq(Ewm+H&K2rz($BIk*oin;OAR>B!J-Z9Kx+!&>o^cP!w+|r~i_}-1 z6m}}h=4zq9z$tSBF!q- ztHW1CU@i$-dXdMK7QR+kF>q3njpMY88b<4jHF5})(z-&Y!;E#uQd6rb)&!!^uIspU z0y+W)t{|)Q66((a4kJpM`Vlo`IRIJFZ=v^R?`T&T^BQ08R)ueom$i(;B$OI5z|Bei&CXFa!D4wy>K zsPOS{CcwicOo!Zigv&y z_yC_`PeA%>EoCC1yL|>s+afxLQk4#j44%S_!HWNg`Ee^OR03fS&hkc3N#ZIA=8^!? z#>2Asb1pXj@Izd_@YKkcnTV-l9*O~+1K$mDO^T6x7A-pkZ5~A#p+rmDiWoN64~&Kp z3IUL*%i!qi-rfVtC5^aaRpxqKZQ<0l!fC94zbY*)m78iPHL&AmA&HSm0EjEJM!R7V z4Kvkq$_@(t`tIOxXr3n_ZX|x<#-fRC+W=+QpHBoUv_oyep|NtjUv5JmU%*DvNUF`m zl3lIGY0csu**m~ncG?4=A`PY((zN31aGigUH^&i|_3Lj$9)o)cnI$i967`wQ?wSeZ zFSUVxW#&22_9L)8r2X>@6;Hg1gJ)`kfUy!G_; zhx{ihXH3T#5_U*1S$LuX1vV9Zs&wuh7IBhkJOD`q7u2KEvVW0wEct zA>k5%lBsNj)Y1rx5O_}wo_f^N_b_UcHH4ItxTzDJpE3iC4vg$4TX$tqTHbd-&8rRf zm%8~R&Gl{z+G?O&HgyS4)g8kxAs=YASz(*dft2DXAkGrJ5xhKGAe07fMsapDC}n@d z1HVt+b0CAi9=Qk&tWk8uW+vWU3a7NX4_DQ@L#NUe@HUM%6vGGU4_*!PI6)r2a#DhqB7e&65bBa}y#V#j-Ah~7HYEOqx8hmmRb1Stt�-HD zWpcHHLnS=HN4SCbrQ1*tsOjJN%#6(1iWs0Ox{sRv5j$hHU{>`*(B+0lC(wE{5PW-U znal2kW3?Sh!`+3-(2ApDS2&*w zaCACOtpdn^K#22Rc%)mSt+@fX1Y2So&?;c$nw5!w$|4DFBx?k|z49=)Z;CLXY@_(< zrhwlY5UtvyG>3Z-%gTtw>fD)S(&Y$*2nBiznn23RdC}4rh>5nIU@+v8VN#<3qp}=Q zP9t~_UBbR^qb3EnLa`SaA#ekOCACro*x{dP*E>GLLi+4_gnk7(Kdlmz?sCMBpu9cA z*qMQ>89HQ1wPX1EddiOAkD`;72214uU2LN?0LsgUA&HKYA`1Xb^OlLYi=K(4v@( zJd|O2>cNyq@{oA13pKBP9A|81u%U99Cl(cZLUXknCSET#x4fDGSZ?fqL<*DCrrIr3=% zJNehmr35YC&oC(Czn>;hR)?{zs6D_qbH1Vl#{!ye4|NTVa(SF;L?KXf4qT+QmV{L^Kh(LO|~~a^!ve_T6?NEh99zn9?7J)l3RV`L~X z1-14DE~zLPg*>#U5Oh_ZnpfUnzIgqCSmsj8kW3x$QmT=un&eT^7D|I z{_dD5|CQ`;loOgoPsXPMSJ;OpfmsEgw!7cbr4N5cYsWEQXi$;aDZYUf|FAkyYae{S zjMKf?#e|~($V^1876ul;QO$6fk>i76260B&Q6@)BjAit-D)o?9ltD$`+HdK3Ee~J$ z?US`vVd>4C8=#4$SNris_XHY>WOw1{_>y)3Ssp>x z6Mv58`Cb(PW;=>*^SOcPdCu?l7*Ao|b_w(OA{3V^lh6=b17tFSWLQ-!@Ba|cRC z%gq2JfG@yGJ_HLP^#`8m(K;vUtT6#E0E9|LJVbHl0EH%d2}VC&kfH4{`VsgmR=hJD zY2f0yNsO~hY0R-Ic*g`n;Z?7Y?G1}#GM;H?9c-r1f_(eXB1F+bwSpi6Z4?iz1tK4i z+@lg%_#Qz76oLTZv^6#vnVo=|(#!xuD<~`s-?HxwY>$opemS}gxdj5_zaMihMv2}o zS|Sk1DuP-QhAS`vD9n{K?1k@TcThSINdo%<0bZShEDY%)7zi4AbAUP@C^&ZxToKkR z<^`bvQi-M`Ndw3iq+WPXaI(NJzXXs7Zfp~(-Jf(ISiBekThm|rASOcG)l4fEWJs8p zoMqEPECDibVmU!&-H1~W&}I#4TV(z;5vLbfYOc{^qYS`Xlsky7o`~R9&S^!iBFKVX zu)IQoT%ZX+jRNz9W6PrHpVNi!qh5ILo#c8?Gd5_^$@5rFeR-bk;rYh^yAh6!s7@XZ zL!|D#1lT+O#U4eRxV$Djkg~h4{B~)^Do0Pf-g;XLY>0v%%ik{WLYB@ zA#Xv)6c2R)E53$QXy9YuVG+Fp|)Sf+rcp z)eGU+-g@2_=&0x}cW)J@Zv<{X{;}C0$S6zeU2b-EB>X0ne};$0w8-}a3Q1iPS|fJ+ za!rhx?;tu*i)KX`fB?VN(CPCmJcbdl&e^#Rv=I*79Q;mMR+AwdXCKQ3?Wpy?VUn4W zGBnTik!CFBJ6!}`{zimcCRGwWCrT!zqa3I}bn!e9Ck^NZ@k0a+x80c|x77`EO@hHT z#sdJ1k?DEc;52EM;NhpkX-Py2&MY()(TpcOgWqqUloj6!uy`0yN-2!ve{QL(E?8pvjj$I>_-J7*7 z1dr!=2AZp02gtd2WOyqqL@P+&ExUty8k~XHg)+YP@@_d5ZZ8&-j6svS3ANH)(7g_D zC83zY&MXuLS=PN)OdNELFua;&6x-%`x&WyEwLUz;3h?W5pvPx0R8H2M=-UR~8UKU= zj409Q&kyWGIQwUc*@e53a$9EB!nfJ1udiRE9Ai+NfGJxNri&@cd@ccf-7)%iI^eTL zs0$)#gPtLTVSbyBBw}f7niNEw-B4+5FSaJ4gn%G z17Kk418U-m5EUeN@P6kP7KQ;644OE3>t^#fmAM-vzDPc1%!K-L5F8VkCI1(egSgD) z7~c>f&AXB?``wo@9~a_c6*kY2p4>AKB!T`oCVW~rmv{RafI@JpS?Kae;T

1yHw(>;H{7h#8^`y>nEBChs<|BM|e1NmE|}n>}F3_fS5z1w3^oY^LeS6ZX3l z&-9_2PK64W3?yTqz}$iU=W#@#Aed}Vi-$WT(QGfpO$K(S3VRzxM-W-psM!wC`GBxX zP6?T|*Vl`#-Z3-Ic7Zk++mW6C&T1T2ctwU#8<9Jw5290!wc!b}d zs$J1N&}6I{zcOM+1dn$cdPmBsfkEU5CV9k1Y$C0~@r~Y{h~>8Aq~1=?`>L<4t!*PN zedJTnqGZg(n^gF|ij2egdl6bo(TvEGqR?L)kqnEBR7sf$f2^qJ%!qoIf3U{c*(3{Z zmY9FUVAT?sEJtvwY%2o75u1%6tqai3AA{ZQm+Ghr7EN)u&j<&it^=lIwu*z8@;WD{ zGS!0+E%ycX@8POWsxae40dt=X@C4Ph5nLqVFqqI4pdR^>{T9O;b%F9VoH`k2y{8Mc zADEdVRPhAV5^g8zo~Siuft8{L3%>yAp)wFP1I~jIo6-$1U2Vt4K;z}?{CgR4_=A+( zvhop+YsGingX*G|Z9XQ-9JxV4Y(5@bh$Qx)^JN$*b<$ck@zkkMzI#xfXT6N~#S0fW z_6huZD1+JPnI^tD%=M((r5^HO&L6P!JhWl2VSL-#e15Iq*t?A)&QGij9i=m;Z=iE? z+dSsK$-a-3`FF8^ssLnpK&fYu{~AkbMFuyKNNRi+_Z(__i|CbKa<+jy-pjy+?<#Ck zLM-ayz@%P++{qYyeUV<`{?tprE6PpHdJrCce5^I8@irFghWEKfKh->)pW$wx){j zGW5I$fDEWpQtX?X@r=%TjxClb<+7k#vCc5W;H`&Lm7gjDlJdT(h_ASdgGsA`&)r6` zZ0cdduF?}!1&mI(mC+DZ!i8M{%!YCH-%X;crxAPx7rXqxEL3-`@V#PczRU0uhLgy~ z@oU+Za{9e7^spk%KM;pdY#~G;)){_i-L){>Effen1X{~%ZEwp2jb%{hYcH^b48p}meH|5`!nrxtZ6JJVSkqc z3V=Ie8^~Eqr!~`fg=qhOul!7oa@q2}4!F)yjZd@LxlmlE0y0QO!}S>5Yc#h7$P%@a zlzL0b;pmJ}U2W|km4OqS%6?pCsNC>Eb`MH~0cLng4S^vroFTspBi4#ji!+IshoD^6 zBt6drsY@C*s>yur41C>6$)jwGq&!a&Ob`eMK^sVRi>4GjK%klr;cgTH^8OH<_9M`a`-_t_vzFWj#^qA@UPtci|h~;~TA173J+` z;u{Z0o-~VI1Zf=@SuN|>4vffmv1Eo#u$D-f0Gs_~aH7y%F0KsAj4zARH)zd>SFBth+bif`5uI#MQjjL9D;$+)#Dt%!!x*wIE zXKN9i2~_a}p$fAKZ$zRejJ^pLYZz<|jY|l^v5Gz;1BK{Go2E2A*Bci)Bz8<(jTs0Y z7x3CESY5j)Z33i;?0ZzFzc3P^L7`wUj6~6AvakY-Gwc5rbZ4{dmS6?Q=Xu_df5voO z{xkr(h6liM9+7MfeKGS1I^JqzdDmlD`wk+CScKwr<70L3M5p++aVp1Bc*vymyihTo z*1eN>Iv<`Tf(a!hR3k?8`DOu<1jRU3>#8}sa`KRDj51E4C9M(IS(l%W7OdpI;mp$W={s1o}F-NHK+h>=w4*crYAZIUSL4JT{ zgJ?z=JLwalcRKbz1m=~SXL=y=WM*46&601_gvpVa^Zv(d5vPc!tqXY&aKFYop&{gN z2kyirNeTMh@YX(ed>mnDnVEHfCdq&aFh#Zm;bS;!rlCxnXMD9o9Rv|H6^GP)L@3^3 zbWK6QAvlnzXnIu%5MZjH*eiRStCryy0ZRJe=V)uxsPe=9oAKQZcx~X|i=Uw^AAxnczQXTuxk{=mIqW$H&u7XTc2 z@ZrPkSAJ;-#!8w#wmpP*vZ&chP0HKF#pNUW#JvRDDOCgg!TkziEVe%T2phvJ#Y~-9 zeR#{IC8t+qt)IPGTdBHtkPPWxl8_~F`RY~PZI`p_3hWD-^Pj$-zVu#On`-j1gMqi= zsy*KX_nSHuG>^!&IPiBK=EfI&ZGS1_#$2#GZ3Tb-;)da2Zg+QgkKcj^4<0B_o2Hdg za|!&2wT%s4!}Hg%hY1he1la=ZBSGtQlG$>WC!6~M-|jJ9RPe%EOS$9uKuX|n2-pJ-%TT#%wq1OG%(21655leZzo_~&}wj|?&Rbq%hHW3ewU^)CaIuiR@lnFAH9E;*<8P=57|f}9(<=Jc z7BnwR*u`YG=C$lppQ@oTy0N#dxdIFtUnt?Tt7Jncqh%iJ^2K%Q*6EZweGKa`)U@Wg z6|2p=a(!Jv^UQ=@li00=jUF8T(ze@ufeWYbJKy7j&Fq!bzO9q{nFJe_G1l|`qC5czWxYZ z(qNd|8y6RMzh&n)TrH0m+`9Kf-Rp(Z;vST1ndtI@T?(4Laz$@LL~pOkv`a||1YeF9 zriIRy6K>k%rxZFL!?JHp8@OSe4&~mb{cryz|(r1TU^> zGjR`O6>BgPclF#ZUnnQMo?P>4X5yVKOI{+|XjFY)N)Q^>2v*e|ro8}5TyE`uy>|MCZjOXS*TYSD_L9@PQ zXhT8-vua>W`%lv`-gwlie)Xa4wTm4JnvW+&Y_6_I%gESY;Qvwc6MkrYy^}J3J{p<> zlcs9(<#H`fC-m(xzHVjzrPWLzs7`wesA{E+{g+0sk{=q?6?5FO?)(1o>i=!tfs4sF zc|dCMs*CGJ$4*_(mn-AOOLl~||2!(*tHu6&Z(WA}b>rT}w~LGavyx6{S>`zCysGE+(r*o8-66q^6dKAE?(}}B~vun!jGG3KS(Z3`Xy=TAUAcH zk&)5eckkZGTY|q<)zjXWl7r(p%{EM%l2bi>tn)WW#l9`!D92aRRxUfd$Ps zwT@)BdX+raF}y05IJ$cF8Nr-&b|9#m$VTEHrm zqa8fulMOc9WBu2e{@3GTW82o~r7RSbiCOkotyr-lH%jwHW3Bg?BZ4JG#_CdcHDt^` zrGcND$2R4&{L*L4m|=L%;oXe>-~E#Nc)5L^#XUn&TN41 zDMw>t%gRM>%cgNPnEU3`X&?EBW2`*KrO5Pf^^7^ZYi77iiGHkV_l{&i^=LqC?c!G% zw{EQ~xX8ZzP%crIgNwLu;X>KeVYLn&O`cm&K!l}x;C7wlAInOe0tGvnmTo61eHV3R z{FFBe(Jxj=?6epEh-H1vrO1)dBF(R!cjDy93G?R7YnR*4xG`=C%P`2ef@SnGW?eym zho|ScnCf|kyx?k5J=B+O@~^)wb3-|C@pku@<0CutZ@u^!+U!-rUvkud7wn)~Wva&S zj6>_JR-w-OQ6<9jDMCYZyV>ijJuf(c^|%|gWAzK~q2AgL(*3^!%qAqZ`#-9^Bekgb z(xrj<D&(t2iYgL5N_BotCxo+eFSaCnJ^Ay3-u()txAS|*t@occ3vO?Vbnjre`p z$ntOPrf%+=iX3!wc|S7nV=rL$sPa3T>q~cpKNS~epKTC7y00Hc-%}JFg@*ciW1h5t zW!SOo%ww+4TpxL#zkFEi(U(bSp*10+b#`a{JzQN+$5)FP@%S;x%g)4P`0pMY*|Dr+ zoMwz%PC!Sefn0gx+dX$m5OG)KzJ0qI+wvY`>K24=Yt@%7&-CAoC;PQxTDWtyrt#QC z68l?gNA%=`LY%p~c>I6yO|t+u+AIBDq0CQN*jxCiebYFlgfIu7E3U&Y@B%`Be7jt@nMMq`qC>aB)7*{>z4rk_x@tlng2Lm1kBT-D8|7G-s%ubI@7IVw)zh z*xRT7`QY2CH=fnHmkkw3yGZ_s#x-_*`9NF z6x=^EVyRKgtry;SF4Gy3iWT0P3b}9e)i#e^FBLDj^m^;XXxG%B{#owp#FKDT@p#t` zve+iOfbCx^F7{bmEb~l0R`OQa&@CK5g$_C;H6i6SwGYaMaw;ZlmMNB09%CKBX?Q3m zEoD~e|2$lBQE|Qhyzsf3<+)=rB<#{!(ueN2y3fkA+y0lFaFWr+fp#ptv74FepCJ>$ j$R=KOK(M6Y$I7MV!P$w+&& Date: Fri, 13 Dec 2024 10:16:49 +0100 Subject: [PATCH 244/489] feat: SP-1856 Adapt filter logic to new settings json schema --- CHANGELOG.md | 2 +- .../{scan_filter.py => file_filters.py} | 174 +++++++++++++----- src/scanoss/scanner.py | 12 +- src/scanoss/scanoss_settings.py | 24 ++- tests/test_scan_filter.py | 24 +-- 5 files changed, 163 insertions(+), 73 deletions(-) rename src/scanoss/{scan_filter.py => file_filters.py} (54%) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8fdb7e4..290dd109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use scanoss.json as default settings file if no argument is supplied - Add —skip-settings-file flag - Update scanoss settings schema to allow skipping specific folders, files, and extensions -- Add ScanFilter class to handle filtering of files and folders based on settings +- Add FileFilters class to handle filtering of files and folders based on settings ## [1.18.1] - 2024-11-19 ### Added diff --git a/src/scanoss/scan_filter.py b/src/scanoss/file_filters.py similarity index 54% rename from src/scanoss/scan_filter.py rename to src/scanoss/file_filters.py index 0633095e..438a74c3 100644 --- a/src/scanoss/scan_filter.py +++ b/src/scanoss/file_filters.py @@ -61,7 +61,7 @@ DEFAULT_SKIPPED_DIR_EXT = { # Folder endings to skip '.egg-info' } -DEFAULT_SKIPPED_EXT = [ # File extensions to skip +DEFAULT_SKIPPED_EXT = { # File extensions to skip '.1', '.2', '.3', @@ -222,12 +222,12 @@ 'manifest', 'sqlite', 'sqlite3', -] +} -class ScanFilter(ScanossBase): +class FileFilters(ScanossBase): """ - Filter for determining which files to process during scanning. + Filter for determining which files to process during scanning, fingerprinting, etc. Handles both inclusion and exclusion rules based on file paths, extensions, and sizes. """ @@ -236,7 +236,7 @@ def __init__( debug: bool = False, trace: bool = False, quiet: bool = False, - scanoss_settings: 'ScanossSettings' = None, + scanoss_settings: 'ScanossSettings | None' = None, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, @@ -255,73 +255,149 @@ def __init__( """ super().__init__(debug, trace, quiet) - self.min_size = 0 - self.max_size = float('inf') self.hidden_files_folders = hidden_files_folders + self.scanoss_settings = scanoss_settings - skip_patterns = [] - - skip_patterns.extend(DEFAULT_SKIPPED_FILES) + self.default_skip_patterns = [] + self.default_skip_patterns.extend(DEFAULT_SKIPPED_FILES) if not all_extensions: - skip_patterns.extend(f'*{ext}' for ext in DEFAULT_SKIPPED_EXT) - skip_patterns.extend(f'*{ext}/' for ext in DEFAULT_SKIPPED_DIR_EXT) + self.default_skip_patterns.extend(f'*{ext}' for ext in DEFAULT_SKIPPED_EXT) + self.default_skip_patterns.extend(f'*{ext}/' for ext in DEFAULT_SKIPPED_DIR_EXT) if not all_folders: - skip_patterns.extend(f'{dir_path}/' for dir_path in DEFAULT_SKIPPED_DIRS) - - if scanoss_settings: - skip_patterns.extend(scanoss_settings.get_skip_patterns()) - self.min_size = scanoss_settings.get_skip_sizes().get('min', 0) - self.max_size = scanoss_settings.get_skip_sizes().get('max', float('inf')) + self.default_skip_patterns.extend(f'{dir_path}/' for dir_path in DEFAULT_SKIPPED_DIRS) - self.skip_patterns = skip_patterns - self.path_spec = PathSpec.from_lines('gitwildmatch', self.skip_patterns) - - def get_filtered_files_from_folder(self, root: str) -> List[str]: + def get_filtered_files_from_folder(self, root: str, operation_type: str) -> List[str]: """Retrieve a list of files to scan or fingerprint from a given directory root based on filter settings. Args: - root (str): Root directory to scan + root (str): Root directory to scan or fingerprint + operation_type (str): Type of operation ('scanning' or 'fingerprinting') Returns: - list[str]: List of files to scan + list[str]: Filtered list of files to scan or fingerprint """ - files = self._walk_with_ignore(root) - return files + skip_patterns = self._get_operation_patterns(operation_type) + path_spec = PathSpec.from_lines('gitwildmatch', skip_patterns) + + filtered_files = [] + for dirpath, _, filenames in self._walk_with_ignore(root): + if self._should_skip_dir(os.path.relpath(dirpath, root)): + continue + + for filename in filenames: + file_path = os.path.join(dirpath, filename) + rel_path = os.path.relpath(file_path, root) - def get_filtered_files_from_files(self, files: List[str]) -> List[str]: + if not self.hidden_files_folders and (filename.startswith('.') or '/.' in rel_path): + continue + + if path_spec.match_file(rel_path): + continue + + try: + file_size = os.path.getsize(file_path) + min_size, max_size = self._get_operation_size_limits(operation_type, file_path) + if min_size <= file_size <= max_size: + filtered_files.append(file_path) + except OSError as e: + self.print_debug(f'Error getting size for {file_path}: {e}') + + return filtered_files + + def get_filtered_files_from_files(self, files: List[str], operation_type: str) -> List[str]: """Retrieve a list of files to scan or fingerprint from a given list of files based on filter settings. Args: - files (List[str]): List of files to scan + files (List[str]): List of files to scan or fingerprint + operation_type (str): Type of operation ('scanning' or 'fingerprinting') Returns: - list[str]: List of files to scan + list[str]: Filtered list of files to scan or fingerprint """ + skip_patterns = self._get_operation_patterns(operation_type) + path_spec = PathSpec.from_lines('gitwildmatch', skip_patterns) + filtered_files = [] - for file in files: - if not self.hidden_files_folders and file.startswith('.'): - self.print_debug(f'Skipping file: {file} (hidden file)') + for file_path in files: + if not os.path.isfile(file_path): continue - file_path = Path(file).resolve() - file_rel_path = file_path.relative_to(Path.cwd()) + filename = os.path.basename(file_path) + if not self.hidden_files_folders and filename.startswith('.'): + continue - if not file_path.exists(): - self.print_debug(f'Skipping file: {file_rel_path} (does not exist)') + if path_spec.match_file(filename): continue - file_size = file_path.stat().st_size + try: + file_size = os.path.getsize(file_path) + min_size, max_size = self._get_operation_size_limits(operation_type, file_path) + if min_size <= file_size <= max_size: + filtered_files.append(file_path) + except OSError as e: + self.print_debug(f'Error getting size for {file_path}: {e}') - if file_size < self.min_size or file_size > self.max_size: - self.print_debug(f'Skipping file: {file} (size: {file_size})') - continue + return filtered_files + + def _get_operation_patterns(self, operation_type: str) -> List[str]: + """Get patterns specific to the operation type, combining defaults with settings. + + Args: + operation_type (str): Type of operation ('scanning' or 'fingerprinting') + + Returns: + List[str]: Combined list of patterns to skip + """ + patterns = self.default_skip_patterns.copy() + + if self.scanoss_settings: + settings_patterns = self.scanoss_settings.get_skip_patterns(operation_type) + if settings_patterns: + patterns.extend(settings_patterns) - if self.path_spec.match_file(str(file_rel_path).lower()): - self.print_debug(f'Skipping file: {file}') + return patterns + + def _get_operation_size_limits(self, operation_type: str, file_path: str = None) -> tuple: + """Get size limits specific to the operation type and file path. + + Args: + operation_type (str): Type of operation ('scanning' or 'fingerprinting') + file_path (str, optional): Path to the file to check against patterns. If None, returns default limits. + + Returns: + tuple: (min_size, max_size) tuple for the given file path and operation type + """ + min_size = 0 + max_size = float('inf') + + if not self.scanoss_settings or not file_path: + return min_size, max_size + + size_rules = self.scanoss_settings.get_skip_sizes(operation_type) + if not size_rules: + return min_size, max_size + + # Convert file path to relative path for pattern matching + try: + rel_path = os.path.relpath(file_path) + except ValueError: + # If file_path is on a different drive, just use the basename + rel_path = os.path.basename(file_path) + + # Check each size rule against the file path + for rule in size_rules: + patterns = rule.get('patterns', []) + if not patterns: continue - filtered_files.append(str(file)) - return filtered_files + # Create a PathSpec for the rule's patterns + path_spec = PathSpec.from_lines('gitwildmatch', patterns) + + # If the file matches any pattern in this rule, use its size limits + if path_spec.match_file(rel_path): + return (rule.get('min', min_size), rule.get('max', max_size)) + + return min_size, max_size def _walk_with_ignore(self, scan_root: str) -> List[str]: files = [] @@ -345,10 +421,12 @@ def _walk_with_ignore(self, scan_root: str) -> List[str]: file_rel_path = rel_path / filename file_size = file_path.stat().st_size - if file_size < self.min_size or file_size > self.max_size: + if file_size < 0 or file_size > float('inf'): self.print_debug(f'Skipping file: {file_rel_path} (size: {file_size})') continue - if self.path_spec.match_file(str(file_rel_path).lower()): + if PathSpec.from_lines('gitwildmatch', self.default_skip_patterns).match_file( + str(file_rel_path).lower() + ): self.print_debug(f'Skipping file: {file_rel_path}') continue else: @@ -361,6 +439,6 @@ def _should_skip_dir(self, dir_rel_path: str) -> bool: is_hidden = dir_path != Path('.') and any(part.startswith('.') for part in dir_path.parts) return ( (is_hidden and not self.hidden_files_folders) - or any(dir_rel_path.lower() == p.rstrip('/').lower() for p in self.skip_patterns) - or self.path_spec.match_file(dir_rel_path.lower() + '/') + or any(dir_rel_path.lower() == p.rstrip('/').lower() for p in self.default_skip_patterns) + or PathSpec.from_lines('gitwildmatch', self.default_skip_patterns).match_file(dir_rel_path.lower() + '/') ) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 9ca793a3..7f01b452 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -33,7 +33,7 @@ from progress.spinner import Spinner from pypac.parser import PACFile -from scanoss.scan_filter import ScanFilter +from scanoss.file_filters import FileFilters from .scanossapi import ScanossApi from .cyclonedx import CycloneDx @@ -185,7 +185,7 @@ def __init__( self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None self._maybe_set_api_sbom() - self.scan_filters = ScanFilter( + self.file_filters = FileFilters( debug=self.debug, trace=self.trace, quiet=self.quiet, @@ -351,7 +351,7 @@ def scan_folder(self, scan_dir: str) -> bool: scan_started = False - to_scan_files = self.scan_filters.get_filtered_files_from_folder(scan_dir) + to_scan_files = self.file_filters.get_filtered_files_from_folder(scan_dir, operation_type='scanning') for to_scan_file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): @@ -595,7 +595,7 @@ def scan_files(self, files: []) -> bool: wfp_file_count = 0 # count number of files in each queue post scan_started = False - to_scan_files = self.scan_filters.get_filtered_files_from_files(files) + to_scan_files = self.file_filters.get_filtered_files_from_files(files, operation_type='scanning') for file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): @@ -995,7 +995,9 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') - to_fingerprint_files = self.scan_filters.get_filtered_files_from_folder(scan_dir) + to_fingerprint_files = self.file_filters.get_filtered_files_from_folder( + scan_dir, operation_type='fingerprinting' + ) for file in to_fingerprint_files: if spinner: spinner.next() diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 4a955385..0e40dac3 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -41,6 +41,12 @@ class BomEntry(TypedDict, total=False): path: str +class SizeFilter(TypedDict, total=False): + patterns: List[str] + min: int + max: int + + class ScanossSettingsError(Exception): pass @@ -269,18 +275,22 @@ def is_legacy(self): """Check if the settings file is legacy""" return self.settings_file_type == 'legacy' - def get_skip_patterns(self) -> List[str]: + def get_skip_patterns(self, operation_type: str) -> List[str]: """ - Get the list of patterns to skip + Get the list of patterns to skip based on the operation type + Args: + operation_type (str): Operation type Returns: List: List of patterns to skip """ - return self.data.get('settings', {}).get('skip', {}).get('patterns', []) + return self.data.get('settings', {}).get('skip', {}).get('patterns', {}).get(operation_type, []) - def get_skip_sizes(self) -> dict: + def get_skip_sizes(self, operation_type: str) -> List[SizeFilter]: """ - Get the min and max sizes to skip + Get the min and max sizes to skip based on the operation type + Args: + operation_type (str): Operation type Returns: - dict: Min and max sizes to skip + List: Min and max sizes to skip """ - return self.data.get('settings', {}).get('skip', {}).get('sizes', {}) + return self.data.get('settings', {}).get('sizes', {}).get(operation_type, []) diff --git a/tests/test_scan_filter.py b/tests/test_scan_filter.py index e77ad266..f2864b68 100644 --- a/tests/test_scan_filter.py +++ b/tests/test_scan_filter.py @@ -3,12 +3,12 @@ import tempfile import unittest -from scanoss.scan_filter import ScanFilter +from scanoss.file_filters import FileFilters -class TestScanFilter(unittest.TestCase): +class TestFileFilters(unittest.TestCase): def setUp(self): - self.scan_filter = ScanFilter(debug=True) + self.file_filters = FileFilters(debug=True) self.test_dir = tempfile.mkdtemp() def tearDown(self): @@ -40,7 +40,7 @@ def test_default_extensions(self): 'dir2/file5.js', ] - filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_default_folders(self): @@ -59,12 +59,12 @@ def test_default_folders(self): 'dir1/file4.go', ] - filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_skip_files_by_size(self): - self.scan_filter.min_size = 150 - self.scan_filter.max_size = 450 + self.file_filters.min_size = 150 + self.file_filters.max_size = 450 files = [ 'file1.js', @@ -80,7 +80,7 @@ def test_skip_files_by_size(self): expected_files = ['file3.py', 'file2.go'] - filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_skip_directories(self): @@ -91,11 +91,11 @@ def test_skip_directories(self): ] self.create_files(files) - self.scan_filter.skip_patterns.append('dir2/') + self.file_filters.skip_patterns.append('dir2/') expected_files = ['file1.js', 'dir1/file2.js'] - filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_custom_skip_patterns(self): @@ -107,11 +107,11 @@ def test_custom_skip_patterns(self): ] self.create_files(files) - self.scan_filter.skip_patterns.append('*.rst') + self.file_filters.skip_patterns.append('*.rst') expected_files = ['file3.py'] - filtered_files = self.scan_filter.get_filtered_files_from_folder(self.test_dir) + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) From c9539b43761f5ddc3a467f2e70f23eb292ce293d Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 13 Dec 2024 11:29:52 +0100 Subject: [PATCH 245/489] feat: SP-1856 Fix relative paths when filtering folders --- src/scanoss/file_filters.py | 122 ++++++++++++++------------------ src/scanoss/scanoss_settings.py | 2 +- tests/test_scan_filter.py | 105 ++++++++++++++++++++------- 3 files changed, 134 insertions(+), 95 deletions(-) diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index 438a74c3..2461ca0c 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -276,40 +276,30 @@ def get_filtered_files_from_folder(self, root: str, operation_type: str) -> List Returns: list[str]: Filtered list of files to scan or fingerprint """ - skip_patterns = self._get_operation_patterns(operation_type) - path_spec = PathSpec.from_lines('gitwildmatch', skip_patterns) + all_files = [] + root_path = Path(root).resolve() - filtered_files = [] - for dirpath, _, filenames in self._walk_with_ignore(root): - if self._should_skip_dir(os.path.relpath(dirpath, root)): + for dirpath, dirnames, filenames in os.walk(root_path): + dirpath = Path(dirpath) + rel_path = dirpath.relative_to(root_path) + + if self._should_skip_dir(str(rel_path), operation_type): + dirnames.clear() continue for filename in filenames: - file_path = os.path.join(dirpath, filename) - rel_path = os.path.relpath(file_path, root) - - if not self.hidden_files_folders and (filename.startswith('.') or '/.' in rel_path): - continue - - if path_spec.match_file(rel_path): - continue - - try: - file_size = os.path.getsize(file_path) - min_size, max_size = self._get_operation_size_limits(operation_type, file_path) - if min_size <= file_size <= max_size: - filtered_files.append(file_path) - except OSError as e: - self.print_debug(f'Error getting size for {file_path}: {e}') + file_path = dirpath / filename + all_files.append(str(file_path)) - return filtered_files + return self.get_filtered_files_from_files(all_files, operation_type, str(root_path)) - def get_filtered_files_from_files(self, files: List[str], operation_type: str) -> List[str]: + def get_filtered_files_from_files(self, files: List[str], operation_type: str, scan_root: str = None) -> List[str]: """Retrieve a list of files to scan or fingerprint from a given list of files based on filter settings. Args: files (List[str]): List of files to scan or fingerprint operation_type (str): Type of operation ('scanning' or 'fingerprinting') + scan_root (str): Root directory to scan or fingerprint Returns: list[str]: Filtered list of files to scan or fingerprint @@ -322,20 +312,35 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str) - if not os.path.isfile(file_path): continue - filename = os.path.basename(file_path) - if not self.hidden_files_folders and filename.startswith('.'): + try: + if scan_root: + rel_path = os.path.relpath(file_path, scan_root) + else: + rel_path = os.path.relpath(file_path) + except ValueError: + # If file_path is broken, symlink ignore it + self.print_debug(f'Ignoring file: {file_path} (broken symlink)') + continue + + if not self.hidden_files_folders and ('/' + '.' in rel_path or rel_path.startswith('.')): + self.print_debug(f'Skipping file: {rel_path} (hidden file)') continue - if path_spec.match_file(filename): + if path_spec.match_file(rel_path): + self.print_debug(f'Skipping file: {rel_path} (matched skip pattern)') continue try: file_size = os.path.getsize(file_path) min_size, max_size = self._get_operation_size_limits(operation_type, file_path) if min_size <= file_size <= max_size: - filtered_files.append(file_path) + filtered_files.append(rel_path) + else: + self.print_debug( + f'Skipping file: {rel_path} (size {file_size} outside limits {min_size}-{max_size})' + ) except OSError as e: - self.print_debug(f'Error getting size for {file_path}: {e}') + self.print_debug(f'Error getting size for {rel_path}: {e}') return filtered_files @@ -377,68 +382,45 @@ def _get_operation_size_limits(self, operation_type: str, file_path: str = None) if not size_rules: return min_size, max_size - # Convert file path to relative path for pattern matching try: rel_path = os.path.relpath(file_path) except ValueError: - # If file_path is on a different drive, just use the basename rel_path = os.path.basename(file_path) - # Check each size rule against the file path for rule in size_rules: patterns = rule.get('patterns', []) if not patterns: continue - # Create a PathSpec for the rule's patterns path_spec = PathSpec.from_lines('gitwildmatch', patterns) - # If the file matches any pattern in this rule, use its size limits if path_spec.match_file(rel_path): return (rule.get('min', min_size), rule.get('max', max_size)) return min_size, max_size - def _walk_with_ignore(self, scan_root: str) -> List[str]: - files = [] - root = Path(scan_root).resolve() + def _should_skip_dir(self, dir_rel_path: str, operation_type: str) -> bool: + """Check if a directory should be skipped based on operation type and patterns. - for dirpath, dirnames, filenames in os.walk(root): - dirpath = Path(dirpath) - rel_path = dirpath.relative_to(root) + Args: + dir_rel_path (str): Relative path to the directory + operation_type (str): Type of operation ('scanning' or 'fingerprinting') - if self._should_skip_dir(str(rel_path)): - self.print_debug(f'Skipping directory: {rel_path}') - dirnames.clear() - continue + Returns: + bool: True if directory should be skipped, False otherwise + """ + dir_path = Path(dir_rel_path) + is_hidden = dir_path != Path('.') and any(part.startswith('.') for part in dir_path.parts) - for filename in filenames: - if not self.hidden_files_folders and filename.startswith('.'): - self.print_debug(f'Skipping file: {filename} (hidden file)') - continue + if is_hidden and not self.hidden_files_folders: + self.print_debug(f'Skipping directory: {dir_rel_path} (hidden directory)') + return True - file_path = dirpath / filename - file_rel_path = rel_path / filename - file_size = file_path.stat().st_size - - if file_size < 0 or file_size > float('inf'): - self.print_debug(f'Skipping file: {file_rel_path} (size: {file_size})') - continue - if PathSpec.from_lines('gitwildmatch', self.default_skip_patterns).match_file( - str(file_rel_path).lower() - ): - self.print_debug(f'Skipping file: {file_rel_path}') - continue - else: - files.append(str(file_rel_path)) + skip_patterns = self._get_operation_patterns(operation_type) + path_spec = PathSpec.from_lines('gitwildmatch', skip_patterns) - return files + if path_spec.match_file(dir_rel_path.lower() + '/'): + self.print_debug(f'Skipping directory: {dir_rel_path} (matched skip pattern)') + return True - def _should_skip_dir(self, dir_rel_path: str) -> bool: - dir_path = Path(dir_rel_path) - is_hidden = dir_path != Path('.') and any(part.startswith('.') for part in dir_path.parts) - return ( - (is_hidden and not self.hidden_files_folders) - or any(dir_rel_path.lower() == p.rstrip('/').lower() for p in self.default_skip_patterns) - or PathSpec.from_lines('gitwildmatch', self.default_skip_patterns).match_file(dir_rel_path.lower() + '/') - ) + return False diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 0e40dac3..21e3e657 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -293,4 +293,4 @@ def get_skip_sizes(self, operation_type: str) -> List[SizeFilter]: Returns: List: Min and max sizes to skip """ - return self.data.get('settings', {}).get('sizes', {}).get(operation_type, []) + return self.data.get('settings', {}).get('skip', {}).get('sizes', {}).get(operation_type, []) diff --git a/tests/test_scan_filter.py b/tests/test_scan_filter.py index f2864b68..ad89ee28 100644 --- a/tests/test_scan_filter.py +++ b/tests/test_scan_filter.py @@ -2,13 +2,15 @@ import shutil import tempfile import unittest +from pathlib import Path from scanoss.file_filters import FileFilters +from scanoss.scanoss_settings import ScanossSettings class TestFileFilters(unittest.TestCase): def setUp(self): - self.file_filters = FileFilters(debug=True) + self.file_filters = FileFilters(debug=True, hidden_files_folders=True) self.test_dir = tempfile.mkdtemp() def tearDown(self): @@ -21,6 +23,10 @@ def create_files(self, files): with open(file_path, 'w') as f: f.write('test') + def get_relative_paths(self, filtered_files): + test_dir_path = Path(self.test_dir).resolve() + return [str(Path(f).resolve().relative_to(test_dir_path)) for f in filtered_files] + def test_default_extensions(self): files = [ 'file2.js', @@ -40,8 +46,8 @@ def test_default_extensions(self): 'dir2/file5.js', ] - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(sorted(filtered_files), sorted(expected_files)) + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) def test_default_folders(self): files = [ @@ -59,12 +65,22 @@ def test_default_folders(self): 'dir1/file4.go', ] - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(sorted(filtered_files), sorted(expected_files)) - - def test_skip_files_by_size(self): - self.file_filters.min_size = 150 - self.file_filters.max_size = 450 + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) + + def test_size_limits(self): + settings = ScanossSettings() + settings.data = { + 'settings': { + 'skip': { + 'sizes': { + 'scanning': [{'patterns': ['*.py'], 'min': 150, 'max': 450}], + 'fingerprinting': [{'patterns': ['*'], 'min': 150, 'max': 450}], + } + } + } + } + file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True) files = [ 'file1.js', @@ -78,27 +94,46 @@ def test_skip_files_by_size(self): with open(file_path, 'w') as f: f.write('a' * (100 if 'file1' in file else 200 if 'file2' in file else 300)) - expected_files = ['file3.py', 'file2.go'] + # For scanning, only *.py files have size limits + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), ['file1.js', 'file2.go', 'file3.py']) - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(sorted(filtered_files), sorted(expected_files)) + # For fingerprinting, all files have size limits + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'fingerprinting') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), ['file2.go', 'file3.py']) - def test_skip_directories(self): + def test_all_extensions(self): + file_filters = FileFilters(debug=True, all_extensions=True, hidden_files_folders=True) files = [ - 'file1.js', - 'dir1/file2.js', - 'dir2/file3.py', + 'file1.txt', + 'file2.md', + 'file3.py', + 'file4.rst', + 'file5.png', ] self.create_files(files) - self.file_filters.skip_patterns.append('dir2/') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(files)) + + def test_all_folders(self): + file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True) + files = [ + '__pycache__/file1.py', + 'nbdist/file2.py', + 'venv/file3.py', + 'normal_dir/file4.py', + ] + self.create_files(files) - expected_files = ['file1.js', 'dir1/file2.js'] + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(files)) - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(sorted(filtered_files), sorted(expected_files)) + def test_custom_patterns(self): + settings = ScanossSettings() + settings.data = {'settings': {'skip': {'patterns': {'scanning': ['*.rst', '*.md', '*.txt']}}}} + file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True) - def test_custom_skip_patterns(self): files = [ 'file1.txt', 'file2.md', @@ -107,12 +142,34 @@ def test_custom_skip_patterns(self): ] self.create_files(files) - self.file_filters.skip_patterns.append('*.rst') + expected_files = ['file3.py'] + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) + + def test_different_patterns_per_operation(self): + settings = ScanossSettings() + settings.data = { + 'settings': {'skip': {'patterns': {'scanning': ['*.rst', '*.md', '*.txt'], 'fingerprinting': ['*.md']}}} + } + file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True) + + files = [ + 'file1.txt', + 'file2.md', + 'file3.py', + 'file4.rst', + ] + self.create_files(files) + # Test scanning patterns expected_files = ['file3.py'] + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) - self.assertEqual(sorted(filtered_files), sorted(expected_files)) + # Test fingerprinting patterns + expected_files = ['file1.txt', 'file3.py', 'file4.rst'] + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'fingerprinting') + self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) if __name__ == '__main__': From 601532910ef931da08ba00a82bdacaaa287cb77b Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 13 Dec 2024 12:00:48 +0100 Subject: [PATCH 246/489] feat: SP-1856 Add debug logs --- src/scanoss/scanner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 7f01b452..2ae5f6bc 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -357,7 +357,7 @@ def scan_folder(self, scan_dir: str) -> bool: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break - self.print_trace(f'Fingerprinting {to_scan_file}...') + self.print_debug(f'Fingerprinting {to_scan_file}...') if spinner: spinner.next() abs_path = Path(scan_dir, to_scan_file).resolve() @@ -608,7 +608,7 @@ def scan_files(self, files: []) -> bool: self.print_trace( f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink if f_size > 0: # Ignore broken links and empty files - self.print_trace(f'Fingerprinting {file}...') + self.print_debug(f'Fingerprinting {file}...') if spinner: spinner.next() wfp = self.winnowing.wfp_for_file(file, file) @@ -1002,6 +1002,7 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): if spinner: spinner.next() abs_path = Path(scan_dir, file).resolve() + self.print_debug(f'Fingerprinting {file}...') wfps += self.winnowing.wfp_for_file(str(abs_path), file) if spinner: spinner.finish() From 7c4972b0d8188688f941396dae9d6b08b190b288 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 13 Dec 2024 12:21:12 +0100 Subject: [PATCH 247/489] feat: SP-1856 Make sure path matching is case insensitive --- src/scanoss/file_filters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index 2461ca0c..d22922f7 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -326,7 +326,7 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str, s self.print_debug(f'Skipping file: {rel_path} (hidden file)') continue - if path_spec.match_file(rel_path): + if path_spec.match_file(rel_path.lower()): self.print_debug(f'Skipping file: {rel_path} (matched skip pattern)') continue @@ -394,7 +394,7 @@ def _get_operation_size_limits(self, operation_type: str, file_path: str = None) path_spec = PathSpec.from_lines('gitwildmatch', patterns) - if path_spec.match_file(rel_path): + if path_spec.match_file(rel_path.lower()): return (rule.get('min', min_size), rule.get('max', max_size)) return min_size, max_size From 2e749409523225691d0d5e863a91b39ec40826b9 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 13 Dec 2024 14:06:59 +0100 Subject: [PATCH 248/489] feat: SP-1856 Fix file_filters tests --- ...st_scan_filter.py => test_file_filters.py} | 158 +++++++++--------- 1 file changed, 83 insertions(+), 75 deletions(-) rename tests/{test_scan_filter.py => test_file_filters.py} (50%) diff --git a/tests/test_scan_filter.py b/tests/test_file_filters.py similarity index 50% rename from tests/test_scan_filter.py rename to tests/test_file_filters.py index ad89ee28..43c4fcf5 100644 --- a/tests/test_scan_filter.py +++ b/tests/test_file_filters.py @@ -2,7 +2,6 @@ import shutil import tempfile import unittest -from pathlib import Path from scanoss.file_filters import FileFilters from scanoss.scanoss_settings import ScanossSettings @@ -23,50 +22,75 @@ def create_files(self, files): with open(file_path, 'w') as f: f.write('test') - def get_relative_paths(self, filtered_files): - test_dir_path = Path(self.test_dir).resolve() - return [str(Path(f).resolve().relative_to(test_dir_path)) for f in filtered_files] - def test_default_extensions(self): files = [ - 'file2.js', - 'file1.go', - 'dir1/file3.py', - 'dir1/file4.go', - 'dir2/file5.js', - 'dir2/file6.png', + 'file1.js', + 'file2.go', + 'file3.py', + 'file4.css', # Should be skipped by default + 'file5.doc', # Should be skipped by default + 'dir1/file6.py', + 'dir1/file7.go', + 'dir2/file8.js', + 'dir2/file9.csv', # Should be skipped by default ] self.create_files(files) expected_files = [ - 'file2.js', - 'file1.go', - 'dir1/file3.py', - 'dir1/file4.go', - 'dir2/file5.js', + 'file1.js', + 'file2.go', + 'file3.py', + 'dir1/file6.py', + 'dir1/file7.go', + 'dir2/file8.js', ] filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) + self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_default_folders(self): files = [ '__pycache__/file1.pyc', - '__pycache__/file2.pyc', + 'venv/file2.py', 'dir1/nbdist/test.py', - 'dir1/nbdist/test1.py', + 'dir1/eggs/test1.py', 'dir1/file3.py', 'dir1/file4.go', + 'dir2/wheels/test.js', + 'dir2/file5.js', + 'package.egg-info/file6.py', ] self.create_files(files) expected_files = [ 'dir1/file3.py', 'dir1/file4.go', + 'dir2/file5.js', ] filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) + self.assertEqual(sorted(filtered_files), sorted(expected_files)) + + def test_default_skipped_files(self): + files = [ + 'gradlew', + 'gradlew.bat', + 'mvnw', + 'license.txt', + 'makefile', + 'normal_file.py', + 'dir1/gradle-wrapper.jar', + 'dir1/normal_file.js', + ] + self.create_files(files) + + expected_files = [ + 'normal_file.py', + 'dir1/normal_file.js', + ] + + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_size_limits(self): settings = ScanossSettings() @@ -83,93 +107,77 @@ def test_size_limits(self): file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True) files = [ - 'file1.js', - 'file2.go', - 'file3.py', + 'file1.js', # 100 bytes + 'file2.py', # 200 bytes - within limits + 'file3.py', # 500 bytes - exceeds max + 'file4.py', # 100 bytes - below min ] - self.create_files(files) for file in files: file_path = os.path.join(self.test_dir, file) with open(file_path, 'w') as f: - f.write('a' * (100 if 'file1' in file else 200 if 'file2' in file else 300)) + if 'file1' in file: + f.write('a' * 100) + elif 'file2' in file: + f.write('a' * 200) + elif 'file3' in file: + f.write('a' * 500) + else: + f.write('a' * 100) # For scanning, only *.py files have size limits filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), ['file1.js', 'file2.go', 'file3.py']) + self.assertEqual(sorted(filtered_files), ['file1.js', 'file2.py']) # For fingerprinting, all files have size limits filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'fingerprinting') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), ['file2.go', 'file3.py']) + self.assertEqual(sorted(filtered_files), ['file2.py']) - def test_all_extensions(self): + def test_all_extensions_flag(self): file_filters = FileFilters(debug=True, all_extensions=True, hidden_files_folders=True) - files = [ - 'file1.txt', - 'file2.md', - 'file3.py', - 'file4.rst', - 'file5.png', - ] - self.create_files(files) - - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(files)) - def test_all_folders(self): - file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True) files = [ - '__pycache__/file1.py', - 'nbdist/file2.py', - 'venv/file3.py', - 'normal_dir/file4.py', + 'file1.js', + 'file2.css', # Would normally be skipped + 'file3.doc', # Would normally be skipped + 'dir1/file4.csv', # Would normally be skipped ] self.create_files(files) filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(files)) + self.assertEqual(sorted(filtered_files), sorted(files)) - def test_custom_patterns(self): - settings = ScanossSettings() - settings.data = {'settings': {'skip': {'patterns': {'scanning': ['*.rst', '*.md', '*.txt']}}}} - file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True) + def test_all_folders_flag(self): + file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True) files = [ - 'file1.txt', - 'file2.md', - 'file3.py', - 'file4.rst', + '__pycache__/file1.py', # Would normally be skipped + 'venv/file2.py', # Would normally be skipped + 'dir1/nbdist/file3.py', # Would normally be skipped + 'dir1/file4.py', ] self.create_files(files) - expected_files = ['file3.py'] filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) - - def test_different_patterns_per_operation(self): - settings = ScanossSettings() - settings.data = { - 'settings': {'skip': {'patterns': {'scanning': ['*.rst', '*.md', '*.txt'], 'fingerprinting': ['*.md']}}} - } - file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True) + self.assertEqual(sorted(filtered_files), sorted(files)) + def test_get_filtered_files_from_files(self): files = [ - 'file1.txt', - 'file2.md', - 'file3.py', - 'file4.rst', + 'file1.js', + 'file2.css', # Should be skipped + 'dir1/file3.py', + 'dir1/__pycache__/file4.py', # Should be skipped ] self.create_files(files) - # Test scanning patterns - expected_files = ['file3.py'] - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) + file_paths = [os.path.join(self.test_dir, f) for f in files] + filtered_files = self.file_filters.get_filtered_files_from_files(file_paths, 'scanning', self.test_dir) - # Test fingerprinting patterns - expected_files = ['file1.txt', 'file3.py', 'file4.rst'] - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'fingerprinting') - self.assertEqual(sorted(self.get_relative_paths(filtered_files)), sorted(expected_files)) + expected_files = [ + 'file1.js', + 'dir1/file3.py', + ] + self.assertEqual(sorted(filtered_files), sorted(expected_files)) if __name__ == '__main__': From 88282fdd8160bbc141826992eebd70ccc42e9c45 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 13 Dec 2024 14:37:27 +0100 Subject: [PATCH 249/489] feat: SP-1856 Add scanoss settings to wfp method --- src/scanoss/cli.py | 22 +++++++++++++++++++++- src/scanoss/scanner.py | 2 +- src/scanoss/scanoss_settings.py | 4 ++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 52b9be8b..2500e817 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -131,6 +131,15 @@ def setup_args() -> None: p_wfp.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, help='Fingerprint the file contents supplied via STDIN (optional)') p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + p_wfp.add_argument( + '--settings', '-st', + type=str, + help='Settings file to use for fingerprinting (optional - default scanoss.json)', + ) + p_wfp.add_argument( + '--skip-settings-file', '-stf', action='store_true', + help='Skip default settings file (scanoss.json) if it exists', + ) # Sub-command: dependency p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], @@ -466,13 +475,24 @@ def wfp(parser, args): if args.output: scan_output = args.output open(scan_output, 'w').close() + + # Load scan settings + scan_settings = None + if not args.skip_settings_file: + scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) + try: + scan_settings.load_json_file(args.settings) + except ScanossSettingsError as e: + print_stderr(f'Error: {e}') + exit(1) scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value # Skip snippet generation or not scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, obfuscate=args.obfuscate, scan_options=scan_options, all_extensions=args.all_extensions, all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm, skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder, - skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet + skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, + scan_settings=scan_settings ) if args.stdin: contents = sys.stdin.buffer.read() diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 2ae5f6bc..16b45d4e 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -129,7 +129,7 @@ def __init__( strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None, - scan_settings: ScanossSettings = None + scan_settings: 'ScanossSettings | None' = None ): """ Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 21e3e657..2343ea8c 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -61,7 +61,7 @@ def __init__( debug: bool = False, trace: bool = False, quiet: bool = False, - filepath: str = None, + filepath: 'str | None' = None, ): """ Args: @@ -99,7 +99,7 @@ def _load_settings_schema(self) -> dict: except Exception as e: raise ScanossSettingsError(f'ERROR: Problem parsing Scanoss Settings Schema JSON file: {e}') from e - def load_json_file(self, filepath: str) -> 'ScanossSettings': + def load_json_file(self, filepath: 'str | None' = None) -> 'ScanossSettings': """ Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. From 898a64b0b1b2575fd3ba75b0e93223d59789581d Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 16 Dec 2024 14:16:51 +0100 Subject: [PATCH 250/489] feat: SP-1856 Skip hidden files --- src/scanoss/file_filters.py | 5 +++ src/scanoss/scanner.py | 75 +++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index d22922f7..9e8d026e 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -332,6 +332,11 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str, s try: file_size = os.path.getsize(file_path) + + if file_size == 0: + self.print_debug(f'Skipping file: {rel_path} (empty file)') + continue + min_size, max_size = self._get_operation_size_limits(operation_type, file_path) if min_size <= file_size <= max_size: filtered_files.append(rel_path) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 16b45d4e..85686f0a 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -601,46 +601,41 @@ def scan_files(self, files: []) -> bool: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break - f_size = 0 - try: - f_size = os.stat(file).st_size - except Exception as e: - self.print_trace( - f'Ignoring missing symlink file: {file} ({e})') # Can fail if there is a broken symlink - if f_size > 0: # Ignore broken links and empty files - self.print_debug(f'Fingerprinting {file}...') - if spinner: - spinner.next() - wfp = self.winnowing.wfp_for_file(file, file) - if wfp is None or wfp == '': - self.print_debug(f'No WFP returned for {file}. Skipping.') - continue - if save_wfps_for_print: - wfp_list.append(wfp) - file_count += 1 - if self.threaded_scan: - wfp_size = len(wfp.encode("utf-8")) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue - if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - scan_block += wfp - scan_size = len(scan_block.encode("utf-8")) - wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue - if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do - scan_started = True - if not self.threaded_scan.run(wait=False): - self.print_stderr( - f'Warning: Some errors encounted while scanning. Results might be incomplete.') - success = False + self.print_debug(f'Fingerprinting {file}...') + if spinner: + spinner.next() + wfp = self.winnowing.wfp_for_file(file, file) + if wfp is None or wfp == '': + self.print_debug(f'No WFP returned for {file}. Skipping.') + continue + if save_wfps_for_print: + wfp_list.append(wfp) + file_count += 1 + if self.threaded_scan: + wfp_size = len(wfp.encode('utf-8')) + # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + scan_block += wfp + scan_size = len(scan_block.encode('utf-8')) + wfp_file_count += 1 + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do + scan_started = True + if not self.threaded_scan.run(wait=False): + self.print_stderr( + f'Warning: Some errors encounted while scanning. Results might be incomplete.' + ) + success = False + # End for loop if self.threaded_scan and scan_block != '': self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted From fe87f90f2aa87f97bca30dc33fba3decf573c42c Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 17 Dec 2024 02:42:52 +0100 Subject: [PATCH 251/489] feat: SP-1856 Use previous logic for default filters instead of pathspec --- src/scanoss/file_filters.py | 99 ++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index 9e8d026e..d5007dde 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -26,7 +26,7 @@ from pathlib import Path from typing import List -from pathspec import PathSpec +from pathspec import GitIgnoreSpec from scanoss.scanoss_settings import ScanossSettings from scanoss.scanossbase import ScanossBase @@ -257,14 +257,8 @@ def __init__( self.hidden_files_folders = hidden_files_folders self.scanoss_settings = scanoss_settings - - self.default_skip_patterns = [] - self.default_skip_patterns.extend(DEFAULT_SKIPPED_FILES) - if not all_extensions: - self.default_skip_patterns.extend(f'*{ext}' for ext in DEFAULT_SKIPPED_EXT) - self.default_skip_patterns.extend(f'*{ext}/' for ext in DEFAULT_SKIPPED_DIR_EXT) - if not all_folders: - self.default_skip_patterns.extend(f'{dir_path}/' for dir_path in DEFAULT_SKIPPED_DIRS) + self.all_extensions = all_extensions + self.all_folders = all_folders def get_filtered_files_from_folder(self, root: str, operation_type: str) -> List[str]: """Retrieve a list of files to scan or fingerprint from a given directory root based on filter settings. @@ -304,9 +298,6 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str, s Returns: list[str]: Filtered list of files to scan or fingerprint """ - skip_patterns = self._get_operation_patterns(operation_type) - path_spec = PathSpec.from_lines('gitwildmatch', skip_patterns) - filtered_files = [] for file_path in files: if not os.path.isfile(file_path): @@ -322,12 +313,7 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str, s self.print_debug(f'Ignoring file: {file_path} (broken symlink)') continue - if not self.hidden_files_folders and ('/' + '.' in rel_path or rel_path.startswith('.')): - self.print_debug(f'Skipping file: {rel_path} (hidden file)') - continue - - if path_spec.match_file(rel_path.lower()): - self.print_debug(f'Skipping file: {rel_path} (matched skip pattern)') + if self._should_skip_file(rel_path, operation_type): continue try: @@ -358,12 +344,10 @@ def _get_operation_patterns(self, operation_type: str) -> List[str]: Returns: List[str]: Combined list of patterns to skip """ - patterns = self.default_skip_patterns.copy() + patterns = [] if self.scanoss_settings: - settings_patterns = self.scanoss_settings.get_skip_patterns(operation_type) - if settings_patterns: - patterns.extend(settings_patterns) + patterns.extend(self.scanoss_settings.get_skip_patterns(operation_type)) return patterns @@ -397,7 +381,7 @@ def _get_operation_size_limits(self, operation_type: str, file_path: str = None) if not patterns: continue - path_spec = PathSpec.from_lines('gitwildmatch', patterns) + path_spec = GitIgnoreSpec.from_lines(patterns) if path_spec.match_file(rel_path.lower()): return (rule.get('min', min_size), rule.get('max', max_size)) @@ -405,7 +389,8 @@ def _get_operation_size_limits(self, operation_type: str, file_path: str = None) return min_size, max_size def _should_skip_dir(self, dir_rel_path: str, operation_type: str) -> bool: - """Check if a directory should be skipped based on operation type and patterns. + """ + Check if a directory should be skipped based on operation type and default rules. Args: dir_rel_path (str): Relative path to the directory @@ -414,18 +399,72 @@ def _should_skip_dir(self, dir_rel_path: str, operation_type: str) -> bool: Returns: bool: True if directory should be skipped, False otherwise """ + dir_name = os.path.basename(dir_rel_path) dir_path = Path(dir_rel_path) - is_hidden = dir_path != Path('.') and any(part.startswith('.') for part in dir_path.parts) - if is_hidden and not self.hidden_files_folders: + if ( + not self.hidden_files_folders + and dir_path != Path('.') + and any(part.startswith('.') for part in dir_path.parts) + ): self.print_debug(f'Skipping directory: {dir_rel_path} (hidden directory)') return True - skip_patterns = self._get_operation_patterns(operation_type) - path_spec = PathSpec.from_lines('gitwildmatch', skip_patterns) + if self.all_folders: + return False + + if dir_name.lower() in DEFAULT_SKIPPED_DIRS: + self.print_debug(f'Skipping directory: {dir_rel_path} (matches default skip directory)') + return True + + for ext in DEFAULT_SKIPPED_DIR_EXT: + if dir_name.lower().endswith(ext): + self.print_debug(f'Skipping directory: {dir_rel_path} (matches default skip extension: {ext})') + return True + + patterns = self._get_operation_patterns(operation_type) + if patterns: + spec = GitIgnoreSpec.from_lines(patterns) + if spec.match_file(dir_rel_path): + self.print_debug(f'Skipping directory: {dir_rel_path} (matches custom pattern)') + return True + + return False + + def _should_skip_file(self, file_rel_path: str, operation_type: str) -> bool: + """ + Check if a file should be skipped based on operation type and default rules. + + Args: + file_rel_path (str): Relative path to the file + operation_type (str): Type of operation ('scanning' or 'fingerprinting') + + Returns: + bool: True if file should be skipped, False otherwise + """ + file_name = os.path.basename(file_rel_path) + + if not self.hidden_files_folders and file_name.startswith('.'): + self.print_debug(f'Skipping file: {file_rel_path} (hidden file)') + return True + + if self.all_extensions: + return False - if path_spec.match_file(dir_rel_path.lower() + '/'): - self.print_debug(f'Skipping directory: {dir_rel_path} (matched skip pattern)') + if file_name.lower() in DEFAULT_SKIPPED_FILES: + self.print_debug(f'Skipping file: {file_rel_path} (matches default skip file)') return True + for ending in DEFAULT_SKIPPED_EXT: + if file_name.lower().endswith(ending): + self.print_debug(f'Skipping file: {file_rel_path} (matches default skip ending: {ending})') + return True + + patterns = self._get_operation_patterns(operation_type) + if patterns: + spec = GitIgnoreSpec.from_lines(patterns) + if spec.match_file(file_rel_path): + self.print_debug(f'Skipping file: {file_rel_path} (matches custom pattern)') + return True + return False From 3636744ea3f0fc997e01f7eb4a7d586983d90d88 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 17 Dec 2024 10:19:54 +0100 Subject: [PATCH 252/489] feat: SP-1856 Fix and add more unit tests --- tests/test_file_filters.py | 129 ++++++++++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 8 deletions(-) diff --git a/tests/test_file_filters.py b/tests/test_file_filters.py index 43c4fcf5..4c844036 100644 --- a/tests/test_file_filters.py +++ b/tests/test_file_filters.py @@ -163,20 +163,133 @@ def test_all_folders_flag(self): def test_get_filtered_files_from_files(self): files = [ - 'file1.js', - 'file2.css', # Should be skipped - 'dir1/file3.py', - 'dir1/__pycache__/file4.py', # Should be skipped + os.path.join(self.test_dir, 'file1.js'), + os.path.join(self.test_dir, 'file2.css'), # Should be skipped + os.path.join(self.test_dir, 'dir1/file3.py'), + os.path.join(self.test_dir, 'dir1/__pycache__/file4.py'), ] self.create_files(files) - file_paths = [os.path.join(self.test_dir, f) for f in files] - filtered_files = self.file_filters.get_filtered_files_from_files(file_paths, 'scanning', self.test_dir) + filtered_files = self.file_filters.get_filtered_files_from_files(files, 'scanning') expected_files = [ - 'file1.js', - 'dir1/file3.py', + os.path.relpath(os.path.join(self.test_dir, 'file1.js'), os.getcwd()), + os.path.relpath(os.path.join(self.test_dir, 'dir1', 'file3.py'), os.getcwd()), + os.path.relpath(os.path.join(self.test_dir, 'dir1', '__pycache__', 'file4.py'), os.getcwd()), + ] + self.assertEqual(sorted(filtered_files), sorted(expected_files)) + + def test_hidden_files_and_folders_enabled(self): + files = [ + '.hidden_file.py', + '.hidden_dir/visible_file.py', + '.hidden_dir/.nested_hidden_file.js', + 'visible_dir/.hidden_file.go', + '.git/config', + '.hidden_dir/nested_dir/.hidden_nested_file.py' + ] + self.create_files(files) + + expected_files = [ + '.hidden_file.py', + '.hidden_dir/visible_file.py', + '.hidden_dir/.nested_hidden_file.js', + 'visible_dir/.hidden_file.go', + '.hidden_dir/nested_dir/.hidden_nested_file.py' + ] + + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(filtered_files), sorted(expected_files)) + + def test_hidden_files_and_folders_disabled(self): + file_filters = FileFilters(debug=True, hidden_files_folders=False) + files = [ + '.hidden_file.py', + '.hidden_dir/visible_file.py', + '.hidden_dir/.nested_hidden_file.js', + 'visible_dir/.hidden_file.go', + 'visible_file.py', + '.git/config' + ] + self.create_files(files) + + expected_files = [ + 'visible_file.py' + ] + + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(filtered_files), sorted(expected_files)) + + def test_all_extensions_mode(self): + file_filters = FileFilters(debug=True, all_extensions=True, hidden_files_folders=True) + files = [ + 'file1.css', + 'file2.doc', + 'file3.csv', + '.hidden_file.dat', + 'dir1/file4.bmp', + 'dir1/.hidden/file5.class', + 'file6.py' + ] + self.create_files(files) + + expected_files = [ + 'file1.css', + 'file2.doc', + 'file3.csv', + '.hidden_file.dat', + 'dir1/file4.bmp', + 'dir1/.hidden/file5.class', + 'file6.py' + ] + + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(filtered_files), sorted(expected_files)) + + def test_all_folders_mode(self): + file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True) + files = [ + '__pycache__/cache.py', + 'venv/lib.py', + 'eggs/module.py', + 'wheels/util.py', + 'normal_dir/file.py', + '.git/config.py' + ] + self.create_files(files) + + expected_files = [ + '__pycache__/cache.py', + 'venv/lib.py', + 'eggs/module.py', + 'wheels/util.py', + 'normal_dir/file.py', + '.git/config.py' + ] + + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + self.assertEqual(sorted(filtered_files), sorted(expected_files)) + + def test_combined_all_modes(self): + file_filters = FileFilters(debug=True, all_extensions=True, all_folders=True, hidden_files_folders=True) + files = [ + '.hidden_dir/file1.css', + '__pycache__/cache.dat', + 'venv/.hidden_file.class', + 'normal_dir/file.py', + '.config/settings.bmp' ] + self.create_files(files) + + expected_files = [ + '.hidden_dir/file1.css', + '__pycache__/cache.dat', + 'venv/.hidden_file.class', + 'normal_dir/file.py', + '.config/settings.bmp' + ] + + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') self.assertEqual(sorted(filtered_files), sorted(expected_files)) From 3f77e7c6d1724aaa75db1e2a1e436c6c23079300 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Thu, 19 Dec 2024 14:24:44 +0000 Subject: [PATCH 253/489] filtering cleanup --- src/scanoss/cli.py | 4 +- src/scanoss/file_filters.py | 229 +++++++++++++++++++------------ src/scanoss/scanner.py | 85 +++++------- src/scanoss/scanoss_settings.py | 45 +++--- src/scanoss/scanpostprocessor.py | 165 +++++++++++----------- src/scanoss/utils/file.py | 48 +++---- tests/test_file_filters.py | 68 ++++++--- 7 files changed, 356 insertions(+), 288 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 2500e817..6bad7753 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -27,9 +27,6 @@ import sys import pypac -from scanoss.utils.file import validate_json_file - - from .inspection.copyleft import Copyleft from .inspection.undeclared_component import UndeclaredComponent from .threadeddependencies import SCOPE @@ -45,6 +42,7 @@ from . import __version__ from .scanner import FAST_WINNOWING from .results import Results +from .utils.file import validate_json_file def print_stderr(*args, **kwargs): diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index d5007dde..0182f948 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -23,14 +23,16 @@ """ import os +import sys from pathlib import Path from typing import List from pathspec import GitIgnoreSpec -from scanoss.scanoss_settings import ScanossSettings -from scanoss.scanossbase import ScanossBase +from .scanoss_settings import ScanossSettings +from .scanossbase import ScanossBase +# Files to skip DEFAULT_SKIPPED_FILES = { 'gradlew', 'gradlew.bat', @@ -45,8 +47,8 @@ 'copying.lib', 'makefile', } - -DEFAULT_SKIPPED_DIRS = { # Folders to skip +# Folders to skip +DEFAULT_SKIPPED_DIRS = { 'nbproject', 'nbbuild', 'nbdist', @@ -58,10 +60,12 @@ 'htmlcov', '__pypackages__', } -DEFAULT_SKIPPED_DIR_EXT = { # Folder endings to skip +# Folder endings to skip +DEFAULT_SKIPPED_DIR_EXT = { '.egg-info' } -DEFAULT_SKIPPED_EXT = { # File extensions to skip +# File extensions to skip +DEFAULT_SKIPPED_EXT = { '.1', '.2', '.3', @@ -232,14 +236,18 @@ class FileFilters(ScanossBase): """ def __init__( - self, - debug: bool = False, - trace: bool = False, - quiet: bool = False, - scanoss_settings: 'ScanossSettings | None' = None, - all_extensions: bool = False, - all_folders: bool = False, - hidden_files_folders: bool = False, + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + scanoss_settings: 'ScanossSettings | None' = None, + all_extensions: bool = False, + all_folders: bool = False, + hidden_files_folders: bool = False, + operation_type: str = 'scanning', + skip_size: int = 0, + skip_extensions = None, + skip_folders = None ): """ Initialize scan filters based on default settings. Optionally append custom settings. @@ -252,47 +260,73 @@ def __init__( all_extensions (bool): Include all file extensions all_folders (bool): Include all folders hidden_files_folders (bool): Include hidden files and folders + operation_type: operation type. can be either 'scanning' or 'fingerprinting' """ super().__init__(debug, trace, quiet) + if skip_folders is None: + skip_folders = [] + if skip_extensions is None: + skip_extensions = [] self.hidden_files_folders = hidden_files_folders self.scanoss_settings = scanoss_settings self.all_extensions = all_extensions self.all_folders = all_folders + self.skip_folders = skip_folders + self.skip_size = skip_size + self.skip_extensions = skip_extensions + self.file_folder_pat_spec = self._get_file_folder_pattern_spec(operation_type) + self.size_pat_rules = self._get_size_limit_pattern_rules(operation_type) - def get_filtered_files_from_folder(self, root: str, operation_type: str) -> List[str]: - """Retrieve a list of files to scan or fingerprint from a given directory root based on filter settings. + def get_filtered_files_from_folder(self, root: str) -> List[str]: + """ + Retrieve a list of files to scan or fingerprint from a given directory root based on filter settings. Args: root (str): Root directory to scan or fingerprint - operation_type (str): Type of operation ('scanning' or 'fingerprinting') Returns: list[str]: Filtered list of files to scan or fingerprint """ + if self.debug: + if self.file_folder_pat_spec: + self.print_stderr(f'Running with {len(self.file_folder_pat_spec)} pattern filters.') + if self.size_pat_rules: + self.print_stderr(f'Running with {len(self.size_pat_rules)} size pattern rules.') + if self.skip_size: + self.print_stderr(f'Running with global skip size: {self.skip_size}') + if self.skip_extensions: + self.print_stderr(f'Running with extra global skip extensions: {self.skip_extensions}') + if self.skip_folders: + self.print_stderr(f'Running with extra global skip folders: {self.skip_folders}') all_files = [] root_path = Path(root).resolve() - + if not root_path.exists() or not root_path.is_dir(): + self.print_stderr(f'ERROR: Specified root directory {root} does not exist or is not a directory.') + return all_files + # Walk the tree looking for files to process. While taking into account files/folders to skip for dirpath, dirnames, filenames in os.walk(root_path): dirpath = Path(dirpath) rel_path = dirpath.relative_to(root_path) + if dirpath.is_symlink(): # TODO should we skip symlink folders? + self.print_msg(f'WARNING: Found symbolic link folder: {dirpath}') - if self._should_skip_dir(str(rel_path), operation_type): + if self._should_skip_dir(str(rel_path)): # Current directory should be skipped dirnames.clear() continue - for filename in filenames: file_path = dirpath / filename all_files.append(str(file_path)) + # End os.walk loop + # Now filter the files and return the reduced list + return self.get_filtered_files_from_files(all_files, str(root_path)) - return self.get_filtered_files_from_files(all_files, operation_type, str(root_path)) - - def get_filtered_files_from_files(self, files: List[str], operation_type: str, scan_root: str = None) -> List[str]: - """Retrieve a list of files to scan or fingerprint from a given list of files based on filter settings. + def get_filtered_files_from_files(self, files: List[str], scan_root: str = None) -> List[str]: + """ + Retrieve a list of files to scan or fingerprint from a given list of files based on filter settings. Args: files (List[str]): List of files to scan or fingerprint - operation_type (str): Type of operation ('scanning' or 'fingerprinting') scan_root (str): Root directory to scan or fingerprint Returns: @@ -300,9 +334,11 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str, s """ filtered_files = [] for file_path in files: - if not os.path.isfile(file_path): + if not os.path.exists(file_path) or not os.path.isfile(file_path) or os.path.islink(file_path): + self.print_debug( + f'WARNING: File {file_path} does not exist, is not a file, or is a symbolic link. Ignoring.' + ) continue - try: if scan_root: rel_path = os.path.relpath(file_path, scan_root) @@ -312,18 +348,14 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str, s # If file_path is broken, symlink ignore it self.print_debug(f'Ignoring file: {file_path} (broken symlink)') continue - - if self._should_skip_file(rel_path, operation_type): + if self._should_skip_file(rel_path): continue - try: file_size = os.path.getsize(file_path) - if file_size == 0: self.print_debug(f'Skipping file: {rel_path} (empty file)') continue - - min_size, max_size = self._get_operation_size_limits(operation_type, file_path) + min_size, max_size = self._get_operation_size_limits(file_path) if min_size <= file_size <= max_size: filtered_files.append(rel_path) else: @@ -332,11 +364,43 @@ def get_filtered_files_from_files(self, files: List[str], operation_type: str, s ) except OSError as e: self.print_debug(f'Error getting size for {rel_path}: {e}') - + # End file loop return filtered_files + def _get_file_folder_pattern_spec(self, operation_type: str = 'scanning'): + """ + Get file path pattern specification. + + :param operation_type: which operation is being performed + :return: List of file path patterns + """ + patterns = self._get_operation_patterns(operation_type) + if patterns: + return GitIgnoreSpec.from_lines(patterns) + return None + + def _get_size_limit_pattern_rules(self, operation_type: str = 'scanning'): + """ + Get size limit pattern rules. + + :param operation_type: which operation is being performed + :return: List of size limit pattern rules + """ + if self.scanoss_settings: + size_rules = self.scanoss_settings.get_skip_sizes(operation_type) + if size_rules: + size_rules_with_patterns = [] + for rule in size_rules: + patterns = rule.get('patterns', []) + if not patterns: + continue + size_rules_with_patterns.append(rule) + return size_rules_with_patterns + return None + def _get_operation_patterns(self, operation_type: str) -> List[str]: - """Get patterns specific to the operation type, combining defaults with settings. + """ + Get patterns specific to the operation type, combining defaults with settings. Args: operation_type (str): Type of operation ('scanning' or 'fingerprinting') @@ -345,63 +409,56 @@ def _get_operation_patterns(self, operation_type: str) -> List[str]: List[str]: Combined list of patterns to skip """ patterns = [] - if self.scanoss_settings: patterns.extend(self.scanoss_settings.get_skip_patterns(operation_type)) - return patterns - def _get_operation_size_limits(self, operation_type: str, file_path: str = None) -> tuple: - """Get size limits specific to the operation type and file path. + def _get_operation_size_limits(self, file_path: str = None) -> tuple: + """ + Get size limits specific to the operation type and file path. Args: - operation_type (str): Type of operation ('scanning' or 'fingerprinting') file_path (str, optional): Path to the file to check against patterns. If None, returns default limits. Returns: tuple: (min_size, max_size) tuple for the given file path and operation type """ min_size = 0 - max_size = float('inf') - - if not self.scanoss_settings or not file_path: + max_size = sys.maxsize + # Apply global minimum file size if specified + if self.skip_size > 0: + min_size = self.skip_size return min_size, max_size - - size_rules = self.scanoss_settings.get_skip_sizes(operation_type) - if not size_rules: + # Return default size limits if no settings specified + if not self.scanoss_settings or not file_path or not self.size_pat_rules: return min_size, max_size - try: rel_path = os.path.relpath(file_path) except ValueError: rel_path = os.path.basename(file_path) - - for rule in size_rules: + rel_path_lower = rel_path.lower() + # Cycle through each rule looking for a match + for rule in self.size_pat_rules: patterns = rule.get('patterns', []) - if not patterns: - continue - - path_spec = GitIgnoreSpec.from_lines(patterns) - - if path_spec.match_file(rel_path.lower()): - return (rule.get('min', min_size), rule.get('max', max_size)) - + if patterns: + path_spec = GitIgnoreSpec.from_lines(patterns) + if path_spec.match_file(rel_path_lower): + return rule.get('min', min_size), rule.get('max', max_size) + # End rules loop return min_size, max_size - def _should_skip_dir(self, dir_rel_path: str, operation_type: str) -> bool: + def _should_skip_dir(self, dir_rel_path: str) -> bool: """ Check if a directory should be skipped based on operation type and default rules. Args: dir_rel_path (str): Relative path to the directory - operation_type (str): Type of operation ('scanning' or 'fingerprinting') Returns: bool: True if directory should be skipped, False otherwise """ dir_name = os.path.basename(dir_rel_path) dir_path = Path(dir_rel_path) - if ( not self.hidden_files_folders and dir_path != Path('.') @@ -409,35 +466,31 @@ def _should_skip_dir(self, dir_rel_path: str, operation_type: str) -> bool: ): self.print_debug(f'Skipping directory: {dir_rel_path} (hidden directory)') return True - if self.all_folders: return False - - if dir_name.lower() in DEFAULT_SKIPPED_DIRS: + dir_name_lower = dir_name.lower() + if dir_name_lower in DEFAULT_SKIPPED_DIRS: self.print_debug(f'Skipping directory: {dir_rel_path} (matches default skip directory)') return True - + if self.skip_folders and dir_name in self.skip_folders: + self.print_debug(f'Skipping directory: {dir_rel_path} (matches skip folder)') + return True for ext in DEFAULT_SKIPPED_DIR_EXT: - if dir_name.lower().endswith(ext): + if dir_name_lower.endswith(ext): self.print_debug(f'Skipping directory: {dir_rel_path} (matches default skip extension: {ext})') return True - patterns = self._get_operation_patterns(operation_type) - if patterns: - spec = GitIgnoreSpec.from_lines(patterns) - if spec.match_file(dir_rel_path): - self.print_debug(f'Skipping directory: {dir_rel_path} (matches custom pattern)') - return True - + if self.file_folder_pat_spec and self.file_folder_pat_spec.match_file(dir_rel_path): + self.print_debug(f'Skipping directory: {dir_rel_path} (matches custom pattern)') + return True return False - def _should_skip_file(self, file_rel_path: str, operation_type: str) -> bool: + def _should_skip_file(self, file_rel_path: str) -> bool: """ Check if a file should be skipped based on operation type and default rules. Args: file_rel_path (str): Relative path to the file - operation_type (str): Type of operation ('scanning' or 'fingerprinting') Returns: bool: True if file should be skipped, False otherwise @@ -447,24 +500,26 @@ def _should_skip_file(self, file_rel_path: str, operation_type: str) -> bool: if not self.hidden_files_folders and file_name.startswith('.'): self.print_debug(f'Skipping file: {file_rel_path} (hidden file)') return True - if self.all_extensions: return False - - if file_name.lower() in DEFAULT_SKIPPED_FILES: + file_name_lower = file_name.lower() + # Look for exact files + if file_name_lower in DEFAULT_SKIPPED_FILES: self.print_debug(f'Skipping file: {file_rel_path} (matches default skip file)') return True - + # Look for file endings for ending in DEFAULT_SKIPPED_EXT: - if file_name.lower().endswith(ending): + if file_name_lower.endswith(ending): self.print_debug(f'Skipping file: {file_rel_path} (matches default skip ending: {ending})') return True - - patterns = self._get_operation_patterns(operation_type) - if patterns: - spec = GitIgnoreSpec.from_lines(patterns) - if spec.match_file(file_rel_path): - self.print_debug(f'Skipping file: {file_rel_path} (matches custom pattern)') - return True - + # Look for custom (extra) endings + if self.skip_extensions: + for ending in self.skip_extensions: + if file_name_lower.endswith(ending): + self.print_debug(f'Skipping file: {file_rel_path} (matches skip extension)') + return True + # Check for file patterns + if self.file_folder_pat_spec and self.file_folder_pat_spec.match_file(file_rel_path): + self.print_debug(f'Skipping file: {file_rel_path} (matches custom pattern)') + return True return False diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 85686f0a..e945742d 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -52,37 +52,11 @@ FAST_WINNOWING = False try: from scanoss_winnowing.winnowing import Winnowing - FAST_WINNOWING = True except ModuleNotFoundError or ImportError: FAST_WINNOWING = False from .winnowing import Winnowing -FILTERED_EXT = [ # File extensions to skip - ".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".ac", ".adoc", ".am", - ".asciidoc", ".bmp", ".build", ".cfg", ".chm", ".class", ".cmake", ".cnf", - ".conf", ".config", ".contributors", ".copying", ".crt", ".csproj", ".css", - ".csv", ".dat", ".data", ".doc", ".docx", ".dtd", ".dts", ".iws", ".c9", ".c9revisions", - ".dtsi", ".dump", ".eot", ".eps", ".geojson", ".gdoc", ".gif", - ".glif", ".gmo", ".gradle", ".guess", ".hex", ".htm", ".html", ".ico", ".iml", - ".in", ".inc", ".info", ".ini", ".ipynb", ".jpeg", ".jpg", ".json", ".jsonld", ".lock", - ".log", ".m4", ".map", ".markdown", ".md", ".md5", ".meta", ".mk", ".mxml", - ".o", ".otf", ".out", ".pbtxt", ".pdf", ".pem", ".phtml", ".plist", ".png", - ".po", ".ppt", ".prefs", ".properties", ".pyc", ".qdoc", ".result", ".rgb", - ".rst", ".scss", ".sha", ".sha1", ".sha2", ".sha256", ".sln", ".spec", ".sql", - ".sub", ".svg", ".svn-base", ".tab", ".template", ".test", ".tex", ".tiff", - ".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".woff", ".woff2", ".xht", - ".xhtml", ".xls", ".xlsx", ".xml", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp", - ".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover", - ".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", - # File endings - "-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice", - "readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3" -] -FILTERED_FILES = { # Files to skip - "gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar", - "thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile" -} WFP_FILE_START = "file=" MAX_POST_SIZE = 64 * 1024 # 64k Max post size @@ -152,6 +126,7 @@ def __init__( self.hpsm = hpsm self.skip_folders = skip_folders self.skip_size = skip_size + self.skip_extensions = skip_extensions ver_details = Scanner.version_details() self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, @@ -184,16 +159,6 @@ def __init__( self.scan_settings = scan_settings self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None self._maybe_set_api_sbom() - - self.file_filters = FileFilters( - debug=self.debug, - trace=self.trace, - quiet=self.quiet, - scanoss_settings=self.scan_settings, - all_extensions=all_extensions, - all_folders=all_folders, - hidden_files_folders=hidden_files_folders, - ) def _maybe_set_api_sbom(self): if not self.scan_settings: @@ -337,6 +302,16 @@ def scan_folder(self, scan_dir: str) -> bool: if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') + file_filters = FileFilters(debug=self.debug, trace=self.trace, quiet=self.quiet, + scanoss_settings=self.scan_settings, + all_extensions=self.all_extensions, + all_folders=self.all_folders, + hidden_files_folders=self.hidden_files_folders, + skip_size=self.skip_size, + skip_folders=self.skip_folders, + skip_extensions=self.skip_extensions, + operation_type='scanning' + ) self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None if not self.quiet and self.isatty: @@ -349,11 +324,9 @@ def scan_folder(self, scan_dir: str) -> bool: file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False - - - to_scan_files = self.file_filters.get_filtered_files_from_folder(scan_dir, operation_type='scanning') - - for to_scan_file in to_scan_files: + + to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) + for to_scan_file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') break @@ -390,7 +363,6 @@ def scan_folder(self, scan_dir: str) -> bool: if not self.threaded_scan.run(wait=False): self.print_stderr('Warning: Some errors encounted while scanning. Results might be incomplete.') success = False - # End for loop if self.threaded_scan and scan_block != '': self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted @@ -583,6 +555,17 @@ def scan_files(self, files: []) -> bool: success = True if not files: raise Exception(f"ERROR: Please provide a non-empty list of filenames to scan") + + file_filters = FileFilters(debug=self.debug, trace=self.trace, quiet=self.quiet, + scanoss_settings=self.scan_settings, + all_extensions=self.all_extensions, + all_folders=self.all_folders, + hidden_files_folders=self.hidden_files_folders, + skip_size=self.skip_size, + skip_folders=self.skip_folders, + skip_extensions=self.skip_extensions, + operation_type='scanning' + ) spinner = None if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') @@ -595,8 +578,7 @@ def scan_files(self, files: []) -> bool: wfp_file_count = 0 # count number of files in each queue post scan_started = False - to_scan_files = self.file_filters.get_filtered_files_from_files(files, operation_type='scanning') - + to_scan_files = file_filters.get_filtered_files_from_files(files) for file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') @@ -983,16 +965,23 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): raise Exception(f'ERROR: Please specify a folder to fingerprint') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') + file_filters = FileFilters(debug=self.debug, trace=self.trace, quiet=self.quiet, + scanoss_settings=self.scan_settings, + all_extensions=self.all_extensions, + all_folders=self.all_folders, + hidden_files_folders=self.hidden_files_folders, + skip_size=self.skip_size, + skip_folders=self.skip_folders, + skip_extensions=self.skip_extensions, + operation_type='scanning' + ) wfps = '' - self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') - to_fingerprint_files = self.file_filters.get_filtered_files_from_folder( - scan_dir, operation_type='fingerprinting' - ) + to_fingerprint_files = file_filters.get_filtered_files_from_folder(scan_dir) for file in to_fingerprint_files: if spinner: spinner.next() diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 2343ea8c..23dd768f 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -29,9 +29,8 @@ import importlib_resources from jsonschema import validate -from scanoss.utils.file import validate_json_file - from .scanossbase import ScanossBase +from .utils.file import validate_json_file DEFAULT_SCANOSS_JSON_FILE = 'scanoss.json' @@ -51,6 +50,25 @@ class ScanossSettingsError(Exception): pass +def _load_settings_schema() -> dict: + """ + Load the SCANOSS settings schema from a JSON file. + + Returns: + dict: The parsed JSON content of the SCANOSS settings schema. + + Raises: + ScanossSettingsError: If there is any issue in locating, reading, or parsing the JSON file + """ + try: + schema_path = importlib_resources.files(__name__) / 'data' / 'scanoss-settings-schema.json' + with importlib_resources.as_file(schema_path) as f: + with open(f, 'r', encoding='utf-8') as file: + return json.load(file) + except Exception as e: + raise ScanossSettingsError(f'ERROR: Problem parsing Scanoss Settings Schema JSON file: {e}') from e + + class ScanossSettings(ScanossBase): """ Handles the loading and parsing of the SCANOSS settings file @@ -70,35 +88,14 @@ def __init__( quiet (bool, optional): Quiet. Defaults to False. filepath (str, optional): Path to settings file. Defaults to None. """ - super().__init__(debug, trace, quiet) self.data = {} self.settings_file_type = None self.scan_type = None - - self.schema = self._load_settings_schema() - + self.schema = _load_settings_schema() if filepath: self.load_json_file(filepath) - def _load_settings_schema(self) -> dict: - """ - Load the SCANOSS settings schema from a JSON file. - - Returns: - dict: The parsed JSON content of the SCANOSS settings schema. - - Raises: - ScanossSettingsError: If there is any issue in locating, reading, or parsing the JSON file - """ - try: - schema_path = importlib_resources.files(__name__) / 'data' / 'scanoss-settings-schema.json' - with importlib_resources.as_file(schema_path) as f: - with open(f, 'r', encoding='utf-8') as file: - return json.load(file) - except Exception as e: - raise ScanossSettingsError(f'ERROR: Problem parsing Scanoss Settings Schema JSON file: {e}') from e - def load_json_file(self, filepath: 'str | None' = None) -> 'ScanossSettings': """ Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 66ad0cac..be35c5a4 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -31,8 +31,52 @@ from .scanossbase import ScanossBase +def _get_match_type_message(result_path: str, bom_entry: BomEntry, action: str) -> str: + """ + Compose message based on match type + + Args: + result_path (str): Path of the scan result + bom_entry (BomEntry): BOM entry to compare with + action (str): Post processing action being performed + + Returns: + str: The message to be printed + """ + if bom_entry.get('path') and bom_entry.get('purl'): + message = f"{action} '{result_path}'. Full match found." + elif bom_entry.get('purl'): + message = f"{action} '{result_path}'. Found PURL match." + else: + message = f"{action} '{result_path}'. Found path match." + return message + + +def _is_full_match(result_path: str, result_purls: List[str], bom_entry: BomEntry) -> bool: + """ + Check if path and purl matches fully with the bom entry + + Args: + result_path (str): Scan result path + result_purls (List[str]): Scan result purls + bom_entry (BomEntry): BOM entry to compare with + + Returns: + bool: True if the path and purl match, False otherwise + """ + if not result_purls: + return False + return bool( + (bom_entry.get('purl') and bom_entry.get('path')) + and (bom_entry.get('path') == result_path) + and (bom_entry.get('purl') in result_purls) + ) + + class ScanPostProcessor(ScanossBase): - """Handles post-processing of the scan results""" + """ + Handles post-processing of the scan results + """ def __init__( self, @@ -76,7 +120,8 @@ def _load_component_info(self): self.component_info_map[purl] = result def post_process(self): - """Post-process the scan results + """ + Post-process the scan results Returns: dict: Processed results @@ -91,11 +136,12 @@ def post_process(self): return self.results def _remove_dismissed_files(self): - """Remove entries from the results based on files and/or purls specified in the SCANOSS settings file""" + """ + Remove entries from the results based on files and/or purls specified in the SCANOSS settings file + """ to_remove_entries = self.scan_settings.get_bom_remove() if not to_remove_entries: return - self.results = { result_path: result for result_path, result in self.results.items() @@ -103,7 +149,9 @@ def _remove_dismissed_files(self): } def _replace_purls(self): - """Replace purls in the results based on the SCANOSS settings file""" + """ + Replace purls in the results based on the SCANOSS settings file + """ to_replace_entries = self.scan_settings.get_bom_replace() if not to_replace_entries: return @@ -133,9 +181,9 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di try: new_component = PackageURL.from_string(to_replace_with_purl).to_dict() new_component_url = purl2url.get_repo_url(to_replace_with_purl) - except Exception: + except RuntimeError: self.print_stderr( - f"Error while replacing: Invalid PURL '{to_replace_with_purl}' in settings file. Abort replacing." + f"ERROR: Issue while replacing: Invalid PURL '{to_replace_with_purl}' in settings file. Skipping." ) return result @@ -159,13 +207,13 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di return result - def _should_replace_result( - self, result_path: str, result: dict, to_replace_entries: List[BomEntry] - ) -> Tuple[bool, str]: - """Check if a result should be replaced based on the SCANOSS settings + def _should_replace_result(self, result_path: str, result: dict, to_replace_entries: List[BomEntry] + ) -> Tuple[bool, str]: + """ + Check if a result should be replaced based on the SCANOSS settings Args: - result_path (str): Path of the result + result_path (str): Path of the result data result (dict): Result to check to_replace_entries (List[BomEntry]): BOM entries to replace from the settings file @@ -181,9 +229,8 @@ def _should_replace_result( if not to_replace_path and not to_replace_purl or not to_replace_with: continue - if ( - self._is_full_match(result_path, result_purls, to_replace_entry) + _is_full_match(result_path, result_purls, to_replace_entry) or (not to_replace_path and to_replace_purl in result_purls) or (not to_replace_purl and to_replace_path == result_path) ): @@ -193,7 +240,14 @@ def _should_replace_result( return False, None def _should_remove_result(self, result_path: str, result: dict, to_remove_entries: List[BomEntry]) -> bool: - """Check if a result should be removed based on the SCANOSS settings""" + """ + Check if a result should be removed based on the SCANOSS settings + + :param result_path: path of the result data + :param result: result to check + :param to_remove_entries: BOM entries to remove from the result + :return: + """ result = result[0] if isinstance(result, list) else result result_purls = result.get('purl', []) @@ -203,9 +257,8 @@ def _should_remove_result(self, result_path: str, result: dict, to_remove_entrie if not to_remove_path and not to_remove_purl: continue - if ( - self._is_full_match(result_path, result_purls, to_remove_entry) + _is_full_match(result_path, result_purls, to_remove_entry) or (not to_remove_path and to_remove_purl in result_purls) or (not to_remove_purl and to_remove_path == result_path) ): @@ -214,75 +267,25 @@ def _should_remove_result(self, result_path: str, result: dict, to_remove_entrie return False - def _print_message( - self, - result_path: str, - result_purls: List[str], - bom_entry: BomEntry, - action: str, - ) -> None: - """Print a message about replacing or removing a result""" + def _print_message(self, result_path: str, result_purls: List[str], bom_entry: BomEntry, action: str) -> None: + """ + Print a message about replacing or removing a result + + :param result_path: + :param result_purls: + :param bom_entry: + :param action: + :return: + """ message = ( - f"{self._get_match_type_message(result_path, result_purls, bom_entry, action)} \n" + f"{_get_match_type_message(result_path, bom_entry, action)} \n" f"Details:\n" f" - PURLs: {', '.join(result_purls)}\n" f" - Path: '{result_path}'\n" ) - if action == 'Replacing': message += f" - {action} with '{bom_entry.get('replace_with')}'" - self.print_debug(message) - - def _get_match_type_message( - self, - result_path: str, - result_purls: List[str], - bom_entry: BomEntry, - action: str, - ) -> str: - """Compose message based on match type - - Args: - result_path (str): Path of the scan result - result_purls (List[str]): Purls of the scan result - bom_entry (BomEntry): BOM entry to compare with - action (str): Post processing action being performed - - Returns: - str: The message to be printed - """ - if bom_entry.get('path') and bom_entry.get('purl'): - message = f"{action} '{result_path}'. Full match found." - elif bom_entry.get('purl'): - message = f"{action} '{result_path}'. Found PURL match." - else: - message = f"{action} '{result_path}'. Found path match." - - return message - - def _is_full_match( - self, - result_path: str, - result_purls: List[str], - bom_entry: BomEntry, - ) -> bool: - """Check if path and purl matches fully with the bom entry - - Args: - result_path (str): Scan result path - result_purls (List[str]): Scan result purls - bom_entry (BomEntry): BOM entry to compare with - - Returns: - bool: True if the path and purl match, False otherwise - """ - - if not result_purls: - return False - - return bool( - (bom_entry.get('purl') and bom_entry.get('path')) - and (bom_entry.get('path') == result_path) - and (bom_entry.get('purl') in result_purls) - ) +# +# End of ScanPostProcessor Class +# \ No newline at end of file diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py index cd68bdac..60838a1d 100644 --- a/src/scanoss/utils/file.py +++ b/src/scanoss/utils/file.py @@ -1,3 +1,26 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" import json import os import sys @@ -5,28 +28,6 @@ from typing import Optional -def print_stderr(*args, **kwargs): - """ - Print the given message to STDERR - """ - print(*args, file=sys.stderr, **kwargs) - - -def is_valid_file(file_path: str) -> bool: - """Check if the specified file exists and is a file - - Args: - file_path (str): The file path - - Returns: - bool: True if valid, False otherwise - """ - if not os.path.exists(file_path) or not os.path.isfile(file_path): - print_stderr(f'Specified file does not exist or is not a file: {file_path}') - return False - return True - - @dataclass class JsonValidation: is_valid: bool @@ -35,7 +36,8 @@ class JsonValidation: def validate_json_file(json_file_path: str) -> JsonValidation: - """Validate if the specified file is indeed a valid JSON file + """ + Validate if the specified file is indeed a valid JSON file Args: json_file_path (str): The JSON file to validate diff --git a/tests/test_file_filters.py b/tests/test_file_filters.py index 4c844036..fef91874 100644 --- a/tests/test_file_filters.py +++ b/tests/test_file_filters.py @@ -1,3 +1,26 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" import os import shutil import tempfile @@ -9,7 +32,7 @@ class TestFileFilters(unittest.TestCase): def setUp(self): - self.file_filters = FileFilters(debug=True, hidden_files_folders=True) + self.file_filters = FileFilters(debug=True, hidden_files_folders=True, operation_type='scanning') self.test_dir = tempfile.mkdtemp() def tearDown(self): @@ -45,7 +68,7 @@ def test_default_extensions(self): 'dir2/file8.js', ] - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_default_folders(self): @@ -68,7 +91,7 @@ def test_default_folders(self): 'dir2/file5.js', ] - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_default_skipped_files(self): @@ -89,7 +112,7 @@ def test_default_skipped_files(self): 'dir1/normal_file.js', ] - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_size_limits(self): @@ -104,8 +127,6 @@ def test_size_limits(self): } } } - file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True) - files = [ 'file1.js', # 100 bytes 'file2.py', # 200 bytes - within limits @@ -125,17 +146,20 @@ def test_size_limits(self): else: f.write('a' * 100) + file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True, operation_type='scanning') + # For scanning, only *.py files have size limits - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), ['file1.js', 'file2.py']) + file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True, operation_type='fingerprinting') + # For fingerprinting, all files have size limits - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'fingerprinting') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), ['file2.py']) def test_all_extensions_flag(self): - file_filters = FileFilters(debug=True, all_extensions=True, hidden_files_folders=True) - + file_filters = FileFilters(debug=True, all_extensions=True, hidden_files_folders=True, operation_type='scanning') files = [ 'file1.js', 'file2.css', # Would normally be skipped @@ -144,11 +168,11 @@ def test_all_extensions_flag(self): ] self.create_files(files) - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(files)) def test_all_folders_flag(self): - file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True) + file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True, operation_type='scanning') files = [ '__pycache__/file1.py', # Would normally be skipped @@ -158,7 +182,7 @@ def test_all_folders_flag(self): ] self.create_files(files) - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(files)) def test_get_filtered_files_from_files(self): @@ -170,7 +194,7 @@ def test_get_filtered_files_from_files(self): ] self.create_files(files) - filtered_files = self.file_filters.get_filtered_files_from_files(files, 'scanning') + filtered_files = self.file_filters.get_filtered_files_from_files(files) expected_files = [ os.path.relpath(os.path.join(self.test_dir, 'file1.js'), os.getcwd()), @@ -198,11 +222,11 @@ def test_hidden_files_and_folders_enabled(self): '.hidden_dir/nested_dir/.hidden_nested_file.py' ] - filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_hidden_files_and_folders_disabled(self): - file_filters = FileFilters(debug=True, hidden_files_folders=False) + file_filters = FileFilters(debug=True, hidden_files_folders=False, operation_type='scanning') files = [ '.hidden_file.py', '.hidden_dir/visible_file.py', @@ -217,7 +241,7 @@ def test_hidden_files_and_folders_disabled(self): 'visible_file.py' ] - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_all_extensions_mode(self): @@ -243,11 +267,11 @@ def test_all_extensions_mode(self): 'file6.py' ] - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_all_folders_mode(self): - file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True) + file_filters = FileFilters(debug=True, all_folders=True, hidden_files_folders=True, operation_type='scanning') files = [ '__pycache__/cache.py', 'venv/lib.py', @@ -267,11 +291,11 @@ def test_all_folders_mode(self): '.git/config.py' ] - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_combined_all_modes(self): - file_filters = FileFilters(debug=True, all_extensions=True, all_folders=True, hidden_files_folders=True) + file_filters = FileFilters(debug=True, all_extensions=True, all_folders=True, hidden_files_folders=True, operation_type='scanning') files = [ '.hidden_dir/file1.css', '__pycache__/cache.dat', @@ -289,7 +313,7 @@ def test_combined_all_modes(self): '.config/settings.bmp' ] - filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir, 'scanning') + filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) From a98bf2a0f1447b16275dec99f5ba770d6c4c76e0 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Mon, 6 Jan 2025 10:14:46 -0300 Subject: [PATCH 254/489] bug:SP-2020 Fixes undeclared component policy check summary * bug:SP-2020 Fixes undeclared component policy check summary --- .gitignore | 1 + CHANGELOG.md | 7 ++++++- src/scanoss/__init__.py | 2 +- src/scanoss/inspection/undeclared_component.py | 8 ++++---- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 6d384760..edeb3710 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ docs/build !tests/data/*.json !docs/source/_static/*.json !scanoss-settings-schema.json +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 290dd109..60deb8f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.19.1] - 2025-01-06 +### Fixed +- Fixed undeclared components inspection + ## [1.19.0] - 2024-11-20 ### Fixed - Check if legacy sbom file before post processing @@ -418,4 +422,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.17.5]: https://github.com/scanoss/scanoss.py/compare/v1.17.4...v1.17.5 [1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 [1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.18.0...v1.18.1 -[1.19.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 \ No newline at end of file +[1.19.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 +[1.19.1]: https://github.com/scanoss/scanoss.py/compare/v1.19.0...v1.19.1 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 74f26e03..4c18d880 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.19.0" +__version__ = "1.19.1" diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index 4e618fdc..af4f0789 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -83,11 +83,11 @@ def _get_summary(self, components: list) -> str: if self.sbom_format == 'settings': summary += (f'Add the following snippet into your `scanoss.json` file\n' f'\n```json\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n```\n') - return summary - - summary += (f'Add the following snippet into your `sbom.json` file\n' + else: + summary += (f'Add the following snippet into your `sbom.json` file\n' f'\n```json\n{json.dumps(self._generate_sbom_file(components), indent=2)}\n```\n') - return summary + + return summary def _json(self, components: list) -> Dict[str, Any]: """ From 8496fc16b16c29e6754f83e738ad69c311c3c06d Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:24:14 +0100 Subject: [PATCH 255/489] Feat/sp 2021 add multiple images (#88) * adding multi-image build --- .github/workflows/container-local-test.yml | 27 +++++++++++++++--- .github/workflows/container-publish-ghcr.yml | 28 +++++++++++++++++-- CHANGELOG.md | 8 +++++- Dockerfile | 4 ++- Makefile | 29 ++++++++++++++------ src/scanoss/__init__.py | 2 +- 6 files changed, 80 insertions(+), 18 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index dfaad526..11f52da1 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -11,6 +11,7 @@ on: - 'main' env: + IMAGE_BASE: scanoss/scanoss-py-base IMAGE_NAME: scanoss/scanoss-py jobs: @@ -38,17 +39,35 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@v3 - # Build Docker image with Buildx - - name: Build Docker Image - id: build-and-push + # Build Docker image with Buildx - Base + - name: Build Docker Image - No Entrypoint + id: build-no-ep + uses: docker/build-push-action@v5 + with: + context: . + push: false + tags: ${{ env.IMAGE_BASE }}:latest + target: no_entry_point + outputs: type=docker,dest=/tmp/scanoss-py-base.tar + + # Build Docker image with Buildx - Entrypoint + - name: Build Docker Image - With Entrypoint + id: build-with-ep uses: docker/build-push-action@v5 with: context: . push: false tags: ${{ env.IMAGE_NAME }}:latest + target: with_entry_point outputs: type=docker,dest=/tmp/scanoss-py.tar - - name: Test Docker Image + - name: Test Docker Image - No Entrypoint + run: | + docker load --input /tmp/scanoss-py-base.tar + docker image ls -a + docker run ${{ env.IMAGE_BASE }} scanoss-py version + + - name: Test Docker Image - With Entrypoint run: | docker load --input /tmp/scanoss-py.tar docker image ls -a diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 42e0c9f9..711d8825 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -9,6 +9,7 @@ on: env: REGISTRY: ghcr.io + IMAGE_NAME_BASE: scanoss/scanoss-py-base IMAGE_NAME: scanoss/scanoss-py jobs: @@ -56,14 +57,36 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} # Extract metadata (tags, labels) for Docker - - name: Extract Docker metadata + - name: Extract Docker metadata - no entrypoint + id: meta-ne + uses: docker/metadata-action@v4 + with: + images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME_BASE }}" + + # Build and push Docker image with Buildx (don't push on PR) + - name: Build and push Docker image - Base (no entrypoint) + id: build-and-push-ne + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta-ne.outputs.tags }} + labels: ${{ steps.meta-ne.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: false + target: no_entry_point + + # Extract metadata (tags, labels) for Docker + - name: Extract Docker metadata - entrypoint id: meta uses: docker/metadata-action@v4 with: images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" # Build and push Docker image with Buildx (don't push on PR) - - name: Build and push Docker image + - name: Build and push Docker image - EP (entrypoint) id: build-and-push uses: docker/build-push-action@v5 with: @@ -75,6 +98,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max provenance: false + target: with_entry_point # Test the docker image - name: Test Published Image diff --git a/CHANGELOG.md b/CHANGELOG.md index 60deb8f5..d7bf15a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.19.2] - 2025-01-06 +### Added +- Add second container image `scanoss-py-base` with no `ENTRYPOINT` + - This is useful for calls from container pipelines (i.e. Jenkins) + ## [1.19.1] - 2025-01-06 ### Fixed - Fixed undeclared components inspection @@ -423,4 +428,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.18.0]: https://github.com/scanoss/scanoss.py/compare/v1.17.5...v1.18.0 [1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.18.0...v1.18.1 [1.19.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 -[1.19.1]: https://github.com/scanoss/scanoss.py/compare/v1.19.0...v1.19.1 \ No newline at end of file +[1.19.1]: https://github.com/scanoss/scanoss.py/compare/v1.19.0...v1.19.1 +[1.19.2]: https://github.com/scanoss/scanoss.py/compare/v1.19.1...v1.19.2 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index bb640bec..b30d6ec8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN tar -xvzf /install/v5.39.210212.tar.gz -C /install \ RUN rm -rf /root/.local/lib/python3.10/site-packages/licensedcode/data/rules /root/.local/lib/python3.10/site-packages/licensedcode/data/cache RUN mkdir /root/.local/lib/python3.10/site-packages/licensedcode/data/rules /root/.local/lib/python3.10/site-packages/licensedcode/data/cache -FROM base +FROM base AS no_entry_point # Copy the Python user packages from the build image to here COPY --from=builder /root/.local /root/.local @@ -56,5 +56,7 @@ WORKDIR /scanoss # Run scancode once to setup any initial files, etc. so that it'll run faster later RUN scancode -p --only-findings --quiet --json /scanoss/scancode-dependencies.json /scanoss && rm -f /scanoss/scancode-dependencies.json +FROM no_entry_point AS with_entry_point + ENTRYPOINT ["scanoss-py"] CMD ["--help"] diff --git a/Makefile b/Makefile index 37043fc0..bdf55662 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ #vars +IMAGE_BASE=scanoss-py-base IMAGE_NAME=scanoss-py REPO=scanoss +DOCKER_FULLNAME_BASE=${REPO}/${IMAGE_BASE} DOCKER_FULLNAME=${REPO}/${IMAGE_NAME} +GHCR_FULLNAME_BASE=ghcr.io/${REPO}/${IMAGE_BASE} GHCR_FULLNAME=ghcr.io/${REPO}/${IMAGE_NAME} VERSION=$(shell ./version.py) @@ -55,15 +58,19 @@ package_all: dist publish ## Build & Publish Python package to PyPI ghcr_build: dist ## Build GitHub container image with local arch @echo "Building GHCR container image..." - docker build -t $(GHCR_FULLNAME) . + docker build --target with_entry_point -t $(GHCR_FULLNAME) . + +ghcr_build_base: dist ## Build GitHub container base image with local arch (no entrypoint) + @echo "Building GHCR base container image..." + docker build --target no_entry_point -t $(GHCR_FULLNAME_BASE) . ghcr_amd64: dist ## Build GitHub AMD64 container image @echo "Building GHCR AMD64 container image..." - docker build -t $(GHCR_FULLNAME) --platform linux/amd64 . + docker build --target with_entry_point -t $(GHCR_FULLNAME) --platform linux/amd64 . ghcr_arm64: dist ## Build GitHub ARM64 container image @echo "Building GHCR ARM64 container image..." - docker build -t $(GHCR_FULLNAME) --platform linux/arm64 . + docker build --target with_entry_point -t $(GHCR_FULLNAME) --platform linux/arm64 . ghcr_tag: ## Tag the latest GH container image with the version from Python @echo "Tagging GHCR latest image with $(VERSION)..." @@ -76,21 +83,25 @@ ghcr_push: ## Push the GH container image to GH Packages ghcr_release: dist ## Build/Publish GitHub multi-platform container image @echo "Building & Releasing GHCR multi-platform container image $(VERSION)..." - docker buildx build --push -t $(GHCR_FULLNAME):$(VERSION) --platform linux/arm64,linux/amd64 . + docker buildx build --push --target with_entry_point -t $(GHCR_FULLNAME):$(VERSION) --platform linux/arm64,linux/amd64 . ghcr_all: ghcr_release ## Execute all GHCR container actions -docker_build: ## Build Docker container image with local arch +docker_build: dist ## Build Docker container image with local arch + @echo "Building Docker image..." + docker build --no-cache --target with_entry_point -t $(DOCKER_FULLNAME) . + +docker_build_base: dist ## Build Docker container image with local arch @echo "Building Docker image..." - docker build --no-cache -t $(DOCKER_FULLNAME) . + docker build --no-cache --target no_entry_point -t $(DOCKER_FULLNAME_BASE) . docker_amd64: dist ## Build Docker AMD64 container image @echo "Building Docker AMD64 container image..." - docker build -t $(DOCKER_FULLNAME) --platform linux/amd64 . + docker build --target with_entry_point -t $(DOCKER_FULLNAME) --platform linux/amd64 . docker_arm64: dist ## Build Docker ARM64 container image @echo "Building Docker ARM64 container image..." - docker build -t $(DOCKER_FULLNAME) --platform linux/arm64 . + docker build --target with_entry_point -t $(DOCKER_FULLNAME) --platform linux/arm64 . docker_tag: ## Tag the latest Docker container image with the version from Python @echo "Tagging Docker latest image with $(VERSION)..." @@ -103,6 +114,6 @@ docker_push: ## Push the Docker container image to DockerHub docker_release: dist ## Build/Publish Docker multi-platform container image @echo "Building & Releasing Docker multi-platform container image $(VERSION)..." - docker buildx build --push -t $(DOCKER_FULLNAME):$(VERSION) --platform linux/arm64,linux/amd64 . + docker buildx build --push --target with_entry_point -t $(DOCKER_FULLNAME):$(VERSION) --platform linux/arm64,linux/amd64 . docker_all: docker_release ## Execute all DockerHub container actions diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 4c18d880..e7251664 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.19.1" +__version__ = "1.19.2" From 31ecd0d277ef8c173175a29edf2dd47ccc1b2868 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:56:42 +0100 Subject: [PATCH 256/489] Feat/sp 2021 add multiple images (#89) - add test retry to release workflow --- .github/workflows/python-local-test.yml | 14 ++++++++++---- .github/workflows/python-publish-pypi.yml | 20 +++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 480e2b04..4ac6b75a 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -33,10 +33,16 @@ jobs: run: make dist - name: Install Test Package - run: | - pip install -r requirements.txt - pip install dist/scanoss-*-py3-none-any.whl - which scanoss-py + uses: nick-fields/retry@v3 + with: + timeout_minutes: 2 + retry_wait_seconds: 10 + max_attempts: 3 + retry_on: error + command: | + pip install -r requirements.txt + pip install dist/scanoss-*-py3-none-any.whl + which scanoss-py - name: Run Tests run: | diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index a8d36608..5cc934e5 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -73,13 +73,19 @@ jobs: python-version: '3.10.x' - name: Install Remote Package - run: | - scanoss_version=$(python ./version.py) - echo "Sleeping before checking PyPI for new release version ${scanoss_version}..." - sleep 60 - echo "Installing scanoss ${scanoss_version}..." - pip install --upgrade scanoss==$scanoss_version - which scanoss-py + uses: nick-fields/retry@v3 + with: + timeout_minutes: 3 + retry_wait_seconds: 10 + max_attempts: 3 + retry_on: error + command: | + scanoss_version=$(python ./version.py) + echo "Sleeping before checking PyPI for new release version ${scanoss_version}..." + sleep 60 + echo "Installing scanoss ${scanoss_version}..." + pip install --upgrade scanoss==$scanoss_version + which scanoss-py - name: Run Tests run: | From 6c55a5dc9f1090be2ffa7e96b3a1da2e7106e433 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:08:27 -0300 Subject: [PATCH 257/489] Chore/groh/add markdown jira md output * chore:SP-2022 Adds Jira Markdown output on inspect command * chore:SP-2023 Adds Jira Markdown output tests * chore:SP-2024 Add Jira Markdown output documentation * Upgrades app version to v1.19.3 --- CHANGELOG.md | 8 +++- CLIENT_HELP.md | 13 ++++++- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 2 +- src/scanoss/inspection/copyleft.py | 28 ++++++++++++++ src/scanoss/inspection/policy_check.py | 32 +++++++++++++++- .../inspection/undeclared_component.py | 38 +++++++++++++++++++ tests/test_policy_inspect.py | 29 +++++++++++++- 8 files changed, 145 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7bf15a4..39a74f87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.19.3] - 2025-01-07 +### Added +- Add Jira Markdown output on inspect command ç +- This is useful for calls from integrations (i.e. Jenkins) + ## [1.19.2] - 2025-01-06 ### Added - Add second container image `scanoss-py-base` with no `ENTRYPOINT` @@ -429,4 +434,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.18.1]: https://github.com/scanoss/scanoss.py/compare/v1.18.0...v1.18.1 [1.19.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 [1.19.1]: https://github.com/scanoss/scanoss.py/compare/v1.19.0...v1.19.1 -[1.19.2]: https://github.com/scanoss/scanoss.py/compare/v1.19.1...v1.19.2 \ No newline at end of file +[1.19.2]: https://github.com/scanoss/scanoss.py/compare/v1.19.1...v1.19.2 +[1.19.3]: https://github.com/scanoss/scanoss.py/compare/v1.19.2...v1.19.3 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 3913303e..3f0cdbb0 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -378,7 +378,18 @@ scanoss-py insp undeclared -i scan-results.json --status undeclared-status.md -- #### Inspect for undeclared components and save results in Markdown format and show status output as sbom.json (legacy) The following command can be used to inspect for undeclared components and save the results in Markdown format. -Default sbom-format 'settings' ```bash scanoss-py insp undeclared -i scan-results.json --status undeclared-status.md --output undeclared.json --format md --sbom-format legacy +``` + +#### Inspect for undeclared components and save results in Jira Markdown format. +The following command can be used to inspect for undeclared components and save the results in Jira Markdown format. +```bash +scanoss-py insp undeclared -i scan-results.json --output undeclared-summary.jiramd --status undeclared-status.jiramd --format jira_md +``` + +#### Inspect for copyleft licenses and save results in Jira Markdown format. +The following command can be used to inspect for undeclared components and save the results in Jira Markdown format. +```bash +scanoss-py insp copyleft -i scan-results.json --output copyleft-summary.jiramd --status copyleft-status.jiramd --format jira_md ``` \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index e7251664..eeabe01c 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.19.2" +__version__ = "1.19.3" diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 6bad7753..fa3b583b 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -334,7 +334,7 @@ def setup_args() -> None: for p in [p_copyleft, p_undeclared]: p.add_argument('-i', '--input', nargs='?', help='Path to results file') - p.add_argument('-f', '--format',required=False ,choices=['json', 'md'], default='json', help='Output format (default: json)') + p.add_argument('-f', '--format',required=False ,choices=['json', 'md', 'jira_md'], default='json', help='Output format (default: json)') p.add_argument('-o', '--output', type=str, help='Save details into a file') p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file') diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index d7d6992f..40da6b2d 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -58,6 +58,7 @@ def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, self.exclude = exclude self.explicit = explicit + def _json(self, components: list) -> Dict[str, Any]: """ Format the components with copyleft licenses as JSON. @@ -100,6 +101,33 @@ def _markdown(self, components: list) -> Dict[str,Any]: 'summary' : f'{len(components)} component(s) with copyleft licenses were found.\n' } + def _jira_markdown(self, components: list) -> Dict[str,Any]: + """ + Format the components with copyleft licenses as Markdown. + + :param components: List of components with copyleft licenses + :return: Dictionary with formatted Markdown details and summary + """ + headers = ['Component', 'Version', 'License', 'URL', 'Copyleft'] + centered_columns = [1, 4] + rows: [[]]= [] + for component in components: + for lic in component['licenses']: + row = [ + component['purl'], + component['version'], + lic['spdxid'], + lic['url'], + 'YES' if lic['copyleft'] else 'NO' + ] + rows.append(row) + # End license loop + # End component loop + return { + 'details': f'{self.generate_jira_table(headers,rows,centered_columns)}', + 'summary' : f'{len(components)} component(s) with copyleft licenses were found.\n' + } + def _filter_components_with_copyleft_licenses(self, components: list) -> list: """ Filter the components list to include only those with copyleft licenses. diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index 20b75f8a..cce8db6a 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -76,7 +76,7 @@ class PolicyCheck(ScanossBase): ScanossBase: A base class providing common functionality for SCANOSS-related operations. """ - VALID_FORMATS = {'md', 'json'} + VALID_FORMATS = {'md', 'json', 'jira_md'} def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, format_type: str = None, status: str = None, output: str = None, name: str = None): @@ -134,6 +134,19 @@ def _markdown(self, components: list) -> Dict[str, Any]: """ pass + @abstractmethod + def _jira_markdown(self, components: list) -> Dict[str, Any]: + """ + Generate Markdown output for the policy check results. + + This method should be implemented by subclasses to create a Markdown representation + of the policy check results. + + :param components: List of components to be included in the output. + :return: A dictionary representing the Markdown output. + """ + pass + def _append_component(self,components: Dict[str, Any], new_component: Dict[str, Any], id: str, status: str) -> Dict[str, Any]: """ @@ -270,6 +283,20 @@ def create_separator(index): table_rows.extend(col_sep + col_sep.join(row) + col_sep for row in rows) return '\n'.join(table_rows) + def generate_jira_table(self, headers, rows, centered_columns=None): + col_sep = '*|*' + if headers is None: + self.print_stderr('ERROR: Header are no set') + return None + + table_header = '|*' + col_sep.join(headers) + '*|\\n' + table = table_header + for row in rows: + if len(headers) == len(row): + table += '|' + '|'.join(row) + '|\\n' + + return table + def _get_formatter(self)-> Callable[[List[dict]], Dict[str,Any]] or None: """ Get the appropriate formatter function based on the specified format. @@ -282,7 +309,8 @@ def _get_formatter(self)-> Callable[[List[dict]], Dict[str,Any]] or None: # a map of which format function to return function_map = { 'json': self._json, - 'md': self._markdown + 'md': self._markdown, + 'jira_md': self._jira_markdown, } return function_map[self.format_type] diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index af4f0789..4b1a2663 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -71,6 +71,26 @@ def _get_undeclared_component(self, components: list)-> list or None: # end component loop return undeclared_components + def _get_jira_summary(self, components: list) -> str: + """ + Get a summary of the undeclared components. + + :param components: List of all components + :return: Component summary markdown + """ + if len(components) > 0: + if self.sbom_format == 'settings': + json_str = (json.dumps(self._generate_scanoss_file(components), indent=2).replace('\n', '\\n') + .replace('"', '\\"')) + return f'{len(components)} undeclared component(s) were found.\\nAdd the following snippet into your `scanoss.json` file\\n{{code:json}}\\n{json_str}\\n{{code}}\\n' + else: + json_str = (json.dumps(self._generate_scanoss_file(components), indent=2).replace('\n', '\\n') + .replace('"', '\\"')) + return f'{len(components)} undeclared component(s) were found.\\nAdd the following snippet into your `sbom.json` file\\n{{code:json}}\\n{json_str}\\n{{code}}\\n' + + return f'{len(components)} undeclared component(s) were found.\\n' + + def _get_summary(self, components: list) -> str: """ Get a summary of the undeclared components. @@ -122,6 +142,24 @@ def _markdown(self, components: list) -> Dict[str,Any]: 'summary': self._get_summary(components), } + def _jira_markdown(self, components: list) -> Dict[str,Any]: + """ + Format the undeclared components as Markdown. + + :param components: List of undeclared components + :return: Dictionary with formatted Markdown details and summary + """ + headers = ['Component', 'Version', 'License'] + rows: [[]]= [] + # TODO look at using SpdxLite license name lookup method + for component in components: + licenses = " - ".join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) + rows.append([component['purl'], component['version'], licenses]) + return { + 'details': f'{self.generate_jira_table(headers,rows)}', + 'summary': self._get_jira_summary(components), + } + def _get_unique_components(self, components: list) -> list: """ Generate a list of unique components. diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 65b0f44b..7e3c073b 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -153,7 +153,7 @@ def test_copyleft_policy_markdown(self): Inspect for undeclared components empty path """ def test_copyleft_policy_empty_path(self): - copyleft = UndeclaredComponent(filepath='', format_type='json') + copyleft = Copyleft(filepath='', format_type='json') success, results = copyleft.run() self.assertTrue(success,2) @@ -332,5 +332,32 @@ def test_undeclared_policy_scanoss_summary(self): self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) + def test_undeclared_policy_jira_markdown_output(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + undeclared = UndeclaredComponent(filepath=input_file_name, format_type='jira_md') + status, results = undeclared.run() + details = results['details'] + summary = results['summary'] + expected_details_output = "|*Component*|*Version*|*License*|\\n|pkg:github/scanoss/scanner.c|1.3.3|BSD-2-Clause - GPL-2.0-only|\\n|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only|\\n|pkg:github/scanoss/wfp|6afc1f6|Zlib - GPL-2.0-only|\\n|pkg:npm/%40electron/rebuild|3.7.0|MIT|\\n|pkg:npm/%40emotion/react|11.13.3|MIT|\\n" + expected_summary_output = r"5 undeclared component(s) were found.\nAdd the following snippet into your `scanoss.json` file\n{code:json}\n{\n \"bom\": {\n \"include\": [\n {\n \"purl\": \"pkg:github/scanoss/scanner.c\"\n },\n {\n \"purl\": \"pkg:github/scanoss/wfp\"\n },\n {\n \"purl\": \"pkg:npm/%40electron/rebuild\"\n },\n {\n \"purl\": \"pkg:npm/%40emotion/react\"\n }\n ]\n }\n}\n{code}\n" + self.assertEqual(status, 0) + self.assertEqual(expected_details_output, details) + self.assertEqual(summary, expected_summary_output) + + def test_copyleft_policy_jira_markdown_output(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = "result.json" + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='jira_md') + status, results = copyleft.run() + details = results['details'] + expected_details_output = r"|*Component*|*Version*|*License*|*URL*|*Copyleft*|\n|pkg:github/scanoss/scanner.c|1.3.3|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/engine|5.4.0|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/wfp|6afc1f6|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/engine|4.0.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n" + self.assertEqual(status, 0) + self.assertEqual(expected_details_output, details) + + + if __name__ == '__main__': unittest.main() \ No newline at end of file From dfaf15b45fa02b28e006041dbd48fa7d928138cb Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Wed, 8 Jan 2025 10:57:50 -0300 Subject: [PATCH 258/489] chore:SP-1022 Refactor on Jira Markdown output --- CHANGELOG.md | 9 ++++- src/scanoss/__init__.py | 2 +- src/scanoss/inspection/policy_check.py | 4 +- .../inspection/undeclared_component.py | 4 +- tests/test_policy_inspect.py | 40 +++++++++++++++++-- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39a74f87..6f4a1c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.19.4] - 2025-01-08 +### Added +- Refactor on Jira Markdown output on inspect command + ## [1.19.3] - 2025-01-07 ### Added -- Add Jira Markdown output on inspect command ç +- Add Jira Markdown output on inspect command - This is useful for calls from integrations (i.e. Jenkins) ## [1.19.2] - 2025-01-06 @@ -435,4 +439,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.19.0]: https://github.com/scanoss/scanoss.py/compare/v1.18.1...v1.19.0 [1.19.1]: https://github.com/scanoss/scanoss.py/compare/v1.19.0...v1.19.1 [1.19.2]: https://github.com/scanoss/scanoss.py/compare/v1.19.1...v1.19.2 -[1.19.3]: https://github.com/scanoss/scanoss.py/compare/v1.19.2...v1.19.3 \ No newline at end of file +[1.19.3]: https://github.com/scanoss/scanoss.py/compare/v1.19.2...v1.19.3 +[1.19.4]: https://github.com/scanoss/scanoss.py/compare/v1.19.3...v1.19.4 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index eeabe01c..d299ab50 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.19.3" +__version__ = "1.19.4" diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index cce8db6a..8f392660 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -289,11 +289,11 @@ def generate_jira_table(self, headers, rows, centered_columns=None): self.print_stderr('ERROR: Header are no set') return None - table_header = '|*' + col_sep.join(headers) + '*|\\n' + table_header = '|*' + col_sep.join(headers) + '*|\n' table = table_header for row in rows: if len(headers) == len(row): - table += '|' + '|'.join(row) + '|\\n' + table += '|' + '|'.join(row) + '|\n' return table diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index 4b1a2663..5d43ff88 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -82,11 +82,11 @@ def _get_jira_summary(self, components: list) -> str: if self.sbom_format == 'settings': json_str = (json.dumps(self._generate_scanoss_file(components), indent=2).replace('\n', '\\n') .replace('"', '\\"')) - return f'{len(components)} undeclared component(s) were found.\\nAdd the following snippet into your `scanoss.json` file\\n{{code:json}}\\n{json_str}\\n{{code}}\\n' + return f'{len(components)} undeclared component(s) were found.\nAdd the following snippet into your `scanoss.json` file\n{{code:json}}\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n{{code}}\n' else: json_str = (json.dumps(self._generate_scanoss_file(components), indent=2).replace('\n', '\\n') .replace('"', '\\"')) - return f'{len(components)} undeclared component(s) were found.\\nAdd the following snippet into your `sbom.json` file\\n{{code:json}}\\n{json_str}\\n{{code}}\\n' + return f'{len(components)} undeclared component(s) were found.\nAdd the following snippet into your `sbom.json` file\n{{code:json}}\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n{{code}}\n' return f'{len(components)} undeclared component(s) were found.\\n' diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 7e3c073b..ef713bd7 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -340,8 +340,36 @@ def test_undeclared_policy_jira_markdown_output(self): status, results = undeclared.run() details = results['details'] summary = results['summary'] - expected_details_output = "|*Component*|*Version*|*License*|\\n|pkg:github/scanoss/scanner.c|1.3.3|BSD-2-Clause - GPL-2.0-only|\\n|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only|\\n|pkg:github/scanoss/wfp|6afc1f6|Zlib - GPL-2.0-only|\\n|pkg:npm/%40electron/rebuild|3.7.0|MIT|\\n|pkg:npm/%40emotion/react|11.13.3|MIT|\\n" - expected_summary_output = r"5 undeclared component(s) were found.\nAdd the following snippet into your `scanoss.json` file\n{code:json}\n{\n \"bom\": {\n \"include\": [\n {\n \"purl\": \"pkg:github/scanoss/scanner.c\"\n },\n {\n \"purl\": \"pkg:github/scanoss/wfp\"\n },\n {\n \"purl\": \"pkg:npm/%40electron/rebuild\"\n },\n {\n \"purl\": \"pkg:npm/%40emotion/react\"\n }\n ]\n }\n}\n{code}\n" + expected_details_output = """|*Component*|*Version*|*License*| +|pkg:github/scanoss/scanner.c|1.3.3|BSD-2-Clause - GPL-2.0-only| +|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only| +|pkg:github/scanoss/wfp|6afc1f6|Zlib - GPL-2.0-only| +|pkg:npm/%40electron/rebuild|3.7.0|MIT| +|pkg:npm/%40emotion/react|11.13.3|MIT| +""" + expected_summary_output = """5 undeclared component(s) were found. +Add the following snippet into your `scanoss.json` file +{code:json} +{ + "bom": { + "include": [ + { + "purl": "pkg:github/scanoss/scanner.c" + }, + { + "purl": "pkg:github/scanoss/wfp" + }, + { + "purl": "pkg:npm/%40electron/rebuild" + }, + { + "purl": "pkg:npm/%40emotion/react" + } + ] + } +} +{code} +""" self.assertEqual(status, 0) self.assertEqual(expected_details_output, details) self.assertEqual(summary, expected_summary_output) @@ -353,7 +381,13 @@ def test_copyleft_policy_jira_markdown_output(self): copyleft = Copyleft(filepath=input_file_name, format_type='jira_md') status, results = copyleft.run() details = results['details'] - expected_details_output = r"|*Component*|*Version*|*License*|*URL*|*Copyleft*|\n|pkg:github/scanoss/scanner.c|1.3.3|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/engine|5.4.0|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/wfp|6afc1f6|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n|pkg:github/scanoss/engine|4.0.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES|\n" + expected_details_output = """|*Component*|*Version*|*License*|*URL*|*Copyleft*| +|pkg:github/scanoss/scanner.c|1.3.3|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| +|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| +|pkg:github/scanoss/engine|5.4.0|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| +|pkg:github/scanoss/wfp|6afc1f6|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| +|pkg:github/scanoss/engine|4.0.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| +""" self.assertEqual(status, 0) self.assertEqual(expected_details_output, details) From 37ceedd5f19029b448c1f137f465438f33a535cc Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 14 Jan 2025 07:58:12 -0300 Subject: [PATCH 259/489] chore:ES-84 Adds scanoss-py Docker image with scanoss user --- CHANGELOG.md | 7 ++++++- Dockerfile | 48 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f4a1c29..5cfc1b18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.19.5] - 2025-01-14 +### Added +- Add Docker image with SCANOSS user + ## [1.19.4] - 2025-01-08 ### Added - Refactor on Jira Markdown output on inspect command @@ -440,4 +444,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.19.1]: https://github.com/scanoss/scanoss.py/compare/v1.19.0...v1.19.1 [1.19.2]: https://github.com/scanoss/scanoss.py/compare/v1.19.1...v1.19.2 [1.19.3]: https://github.com/scanoss/scanoss.py/compare/v1.19.2...v1.19.3 -[1.19.4]: https://github.com/scanoss/scanoss.py/compare/v1.19.3...v1.19.4 \ No newline at end of file +[1.19.4]: https://github.com/scanoss/scanoss.py/compare/v1.19.3...v1.19.4 +[1.19.5]: https://github.com/scanoss/scanoss.py/compare/v1.19.4...v1.19.5 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b30d6ec8..12a7eac8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,12 @@ -FROM --platform=$BUILDPLATFORM python:3.10-slim-buster as base +FROM --platform=$BUILDPLATFORM python:3.10-slim-buster AS base LABEL maintainer="SCANOSS " LABEL org.opencontainers.image.source=https://github.com/scanoss/scanoss.py LABEL org.opencontainers.image.description="SCANOSS Python CLI Container" LABEL org.opencontainers.image.licenses=MIT -FROM base as builder +# Compile and install all the necessary python requirements +FROM base AS builder # Setup the required build tooling RUN apt-get update \ @@ -13,49 +14,64 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN mkdir /install +# Create and activate virtual environment +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + WORKDIR /install -ENV PATH=/root/.local/bin:$PATH # assumes `make dist` as prerequisite COPY ./dist/scanoss-*-py3-none-any.whl /install/ +COPY ./requirements-dev.txt /install/ # Install dependencies -RUN pip3 install --user /install/scanoss-*-py3-none-any.whl -RUN pip3 install --user scanoss_winnowing -RUN pip3 install --user scancode-toolkit-mini -#RUN pip3 install --user typecode-libmagic +RUN pip3 install --no-cache-dir /install/scanoss-*-py3-none-any.whl +RUN pip3 install --no-cache-dir scanoss_winnowing +RUN pip3 install --no-cache-dir -r /install/requirements-dev.txt +RUN pip3 install --no-cache-dir scancode-toolkit-mini # Download compile and install typecode-libmagic from source (as there is not ARM wheel available) ADD https://github.com/nexB/typecode_libmagic_from_sources/archive/refs/tags/v5.39.210212.tar.gz /install/ RUN tar -xvzf /install/v5.39.210212.tar.gz -C /install \ && cd /install/typecode_libmagic_from_sources* \ - && ./build.sh && python3 setup.py sdist bdist_wheel \ - && pip3 install --user `ls /install/typecode_libmagic_from_sources*/dist/*.whl` + && ./build.sh \ + && python3 setup.py sdist bdist_wheel \ + && ls /install/typecode_libmagic_from_sources*/dist/*.whl \ + && pip3 install --no-cache-dir `ls /install/typecode_libmagic_from_sources*/dist/*.whl` + +RUN pip3 uninstall --no-cache-dir -y -r /install/requirements-dev.txt # Remove license data references as they are not required for dependency scanning (to save space) -RUN rm -rf /root/.local/lib/python3.10/site-packages/licensedcode/data/rules /root/.local/lib/python3.10/site-packages/licensedcode/data/cache -RUN mkdir /root/.local/lib/python3.10/site-packages/licensedcode/data/rules /root/.local/lib/python3.10/site-packages/licensedcode/data/cache +RUN rm -rf /opt/venv/lib/python3.10/site-packages/licensedcode/data/rules /opt/venv/lib/python3.10/site-packages/licensedcode/data/cache +RUN mkdir /opt/venv/lib/python3.10/site-packages/licensedcode/data/rules /opt/venv/lib/python3.10/site-packages/licensedcode/data/cache +# Image with no default entry point FROM base AS no_entry_point +# Create scanoss user for compatibility +RUN groupadd -g 1000 scanoss && \ + useradd -u 1000 -g scanoss -m -s /bin/bash scanoss + # Copy the Python user packages from the build image to here -COPY --from=builder /root/.local /root/.local +COPY --from=builder /opt/venv /opt/venv # Setup the path and explicitly set GRPC Polling strategy -ENV PATH=/root/.local/bin:$PATH +ENV PATH=/opt/venv/bin:$PATH ENV GRPC_POLL_STRATEGY=poll +# Install jq and curl commands RUN apt-get update \ && apt-get install -y --no-install-recommends jq curl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -VOLUME /scanoss +# Setup working directory and user WORKDIR /scanoss - +RUN chown -R scanoss:scanoss /scanoss /opt/venv +USER scanoss # Run scancode once to setup any initial files, etc. so that it'll run faster later RUN scancode -p --only-findings --quiet --json /scanoss/scancode-dependencies.json /scanoss && rm -f /scanoss/scancode-dependencies.json +# Image with a default scanoss-py entry point FROM no_entry_point AS with_entry_point ENTRYPOINT ["scanoss-py"] From 01b74da180d76b354cd3d02d7e12f4540bebea37 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 14 Jan 2025 10:18:52 -0300 Subject: [PATCH 260/489] Updgrade scanoss-py version to v1.19.5 --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index d299ab50..4cec5556 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.19.4" +__version__ = "1.19.5" From 045ba3d73c0a17f02280a7a712851caa242d7da3 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 14 Jan 2025 10:27:41 -0300 Subject: [PATCH 261/489] chore:ES-86 Fixes local test --- .github/workflows/container-local-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 11f52da1..9641912c 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -73,7 +73,7 @@ jobs: docker image ls -a docker run ${{ env.IMAGE_NAME }} version docker run ${{ env.IMAGE_NAME }} utils fast - docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests + docker run --user $(id -u):$(id -g) -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then From 5952dfee4bb98a884fd017be72eb11d6ea191001 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Tue, 14 Jan 2025 19:35:17 +0000 Subject: [PATCH 262/489] add separate jenkins image --- .github/workflows/container-local-test.yml | 18 +++++++++++++++ .github/workflows/container-publish-ghcr.yml | 23 ++++++++++++++++++++ Dockerfile | 17 ++++++++++----- Makefile | 10 ++++++++- 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 9641912c..1b266e54 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -13,6 +13,7 @@ on: env: IMAGE_BASE: scanoss/scanoss-py-base IMAGE_NAME: scanoss/scanoss-py + IMAGE_JENKINS: scanoss/scanoss-py-jenkins jobs: build: @@ -50,6 +51,17 @@ jobs: target: no_entry_point outputs: type=docker,dest=/tmp/scanoss-py-base.tar + # Build Docker image with Buildx - Jenkins + - name: Build Docker Image - Jenkins + id: build-je + uses: docker/build-push-action@v5 + with: + context: . + push: false + tags: ${{ env.IMAGE_JENKINS }}:latest + target: jenkins + outputs: type=docker,dest=/tmp/scanoss-py-jenkins.tar + # Build Docker image with Buildx - Entrypoint - name: Build Docker Image - With Entrypoint id: build-with-ep @@ -67,6 +79,12 @@ jobs: docker image ls -a docker run ${{ env.IMAGE_BASE }} scanoss-py version + - name: Test Docker Image - Jenkins + run: | + docker load --input /tmp/scanoss-py-jenkins.tar + docker image ls -a + docker run ${{ env.IMAGE_JENKINS }} scanoss-py version + - name: Test Docker Image - With Entrypoint run: | docker load --input /tmp/scanoss-py.tar diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 711d8825..5edb8585 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -11,6 +11,7 @@ env: REGISTRY: ghcr.io IMAGE_NAME_BASE: scanoss/scanoss-py-base IMAGE_NAME: scanoss/scanoss-py + IMAGE_JENKINS: scanoss/scanoss-py-jenkins jobs: deploy: @@ -78,6 +79,28 @@ jobs: provenance: false target: no_entry_point + # Extract metadata (tags, labels) for Docker + - name: Extract Docker metadata - jenkins + id: meta-je + uses: docker/metadata-action@v4 + with: + images: "${{ env.REGISTRY }}/${{ env.IMAGE_JENKINS }}" + + # Build and push Docker image with Buildx (don't push on PR) + - name: Build and push Docker image - Jenkins + id: build-and-push-je + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta-je.outputs.tags }} + labels: ${{ steps.meta-je.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: false + target: jenkins + # Extract metadata (tags, labels) for Docker - name: Extract Docker metadata - entrypoint id: meta diff --git a/Dockerfile b/Dockerfile index 12a7eac8..1ba624dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,10 +48,6 @@ RUN mkdir /opt/venv/lib/python3.10/site-packages/licensedcode/data/rules /opt/v # Image with no default entry point FROM base AS no_entry_point -# Create scanoss user for compatibility -RUN groupadd -g 1000 scanoss && \ - useradd -u 1000 -g scanoss -m -s /bin/bash scanoss - # Copy the Python user packages from the build image to here COPY --from=builder /opt/venv /opt/venv # Setup the path and explicitly set GRPC Polling strategy @@ -66,11 +62,20 @@ RUN apt-get update \ # Setup working directory and user WORKDIR /scanoss -RUN chown -R scanoss:scanoss /scanoss /opt/venv -USER scanoss # Run scancode once to setup any initial files, etc. so that it'll run faster later RUN scancode -p --only-findings --quiet --json /scanoss/scancode-dependencies.json /scanoss && rm -f /scanoss/scancode-dependencies.json +# Image with no default entry point +FROM no_entry_point AS jenkins + +# Create scanoss user for compatibility +RUN groupadd -g 1000 jenkins && \ + useradd -u 1000 -g jenkins -m -s /bin/bash jenkins + +# Copy the Python user packages from the build image to here +RUN chown -R jenkins:jenkins /scanoss /opt/venv +USER jenkins + # Image with a default scanoss-py entry point FROM no_entry_point AS with_entry_point diff --git a/Makefile b/Makefile index bdf55662..ebde3c57 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,10 @@ ghcr_build_base: dist ## Build GitHub container base image with local arch (no @echo "Building GHCR base container image..." docker build --target no_entry_point -t $(GHCR_FULLNAME_BASE) . +ghcr_build_jenkins: dist ## Build GitHub container jenkins image with local arch + @echo "Building GHCR base container image..." + docker build --target jenkins -t $(GHCR_FULLNAME_BASE) . + ghcr_amd64: dist ## Build GitHub AMD64 container image @echo "Building GHCR AMD64 container image..." docker build --target with_entry_point -t $(GHCR_FULLNAME) --platform linux/amd64 . @@ -91,10 +95,14 @@ docker_build: dist ## Build Docker container image with local arch @echo "Building Docker image..." docker build --no-cache --target with_entry_point -t $(DOCKER_FULLNAME) . -docker_build_base: dist ## Build Docker container image with local arch +docker_build_base: dist ## Build Base Docker container image with local arch - no entrypoint @echo "Building Docker image..." docker build --no-cache --target no_entry_point -t $(DOCKER_FULLNAME_BASE) . +docker_build_jenkins: dist ## Build Jenkins Docker container image with local arch + @echo "Building Docker image..." + docker build --no-cache --target jenkins -t $(DOCKER_FULLNAME_BASE) . + docker_amd64: dist ## Build Docker AMD64 container image @echo "Building Docker AMD64 container image..." docker build --target with_entry_point -t $(DOCKER_FULLNAME) --platform linux/amd64 . From d1732f30660cac21334a23d39678250d8c6d82a3 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Tue, 14 Jan 2025 19:41:20 +0000 Subject: [PATCH 263/489] remove user specification --- .github/workflows/container-local-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 1b266e54..9eeead8c 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -91,7 +91,7 @@ jobs: docker image ls -a docker run ${{ env.IMAGE_NAME }} version docker run ${{ env.IMAGE_NAME }} utils fast - docker run --user $(id -u):$(id -g) -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests + docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then From 0ee9dbbc558ef7681f1c4f8e38774e58ca3f90ea Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz <166400360+ortizjeronimo@users.noreply.github.com> Date: Mon, 20 Jan 2025 09:33:29 -0300 Subject: [PATCH 264/489] updated documentation theme to enable integrated search --- docs/requirements-docs.txt | 2 +- docs/source/conf.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a95ae18b..52b04f2e 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1 +1 @@ -furo +sphinx_rtd_theme \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 1a9058e2..19fd7e9a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,7 +14,7 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = [] +extensions = ['sphinx_rtd_theme'] templates_path = ["_templates"] exclude_patterns = [] @@ -24,7 +24,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'furo' +html_theme = 'sphinx_rtd_theme' html_logo = 'scanosslogo.jpg' html_static_path = ['_static'] From 5b4b8d885e94263dbc986ca96752d9e4b1b31acb Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 30 Jan 2025 12:16:50 +0100 Subject: [PATCH 265/489] feat: ES-148 Omit settings file if it doesn't exist instead of throwing an error --- CHANGELOG.md | 4 ++++ src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 12 ++++++------ src/scanoss/scanoss_settings.py | 8 ++++++-- src/scanoss/utils/file.py | 20 ++++++++++++++++---- 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cfc1b18..4c901c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.19.6] - 2025-01-30 +### Added +- Omit settings file if it does not exist instead of throwing an error. + ## [1.19.5] - 2025-01-14 ### Added - Add Docker image with SCANOSS user diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 4cec5556..758d61eb 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.19.5" +__version__ = "1.19.6" diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index d275ea06..92f87821 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -197,12 +197,12 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'name': 'scanoss-py', 'version': __version__, } - ] - }, - 'component': { - 'type': 'application', - 'name': 'NOASSERTION', - 'version': 'NOASSERTION' + ], + 'component': { + 'type': 'application', + 'name': 'NOASSERTION', + 'version': 'NOASSERTION' + } }, 'components': [], 'vulnerabilities': [] diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 23dd768f..e5819a76 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -30,7 +30,7 @@ from jsonschema import validate from .scanossbase import ScanossBase -from .utils.file import validate_json_file +from .utils.file import JSON_ERROR_FILE_NOT_FOUND, validate_json_file DEFAULT_SCANOSS_JSON_FILE = 'scanoss.json' @@ -114,7 +114,11 @@ def load_json_file(self, filepath: 'str | None' = None) -> 'ScanossSettings': result = validate_json_file(json_file) if not result.is_valid: - raise ScanossSettingsError(f'Problem with settings file. {result.error}') + if result.error_code == JSON_ERROR_FILE_NOT_FOUND: + self.print_debug(f'The provided settings file "{filepath}" was not found. Skipping...') + return self + else: + raise ScanossSettingsError(f'Problem with settings file. {result.error}') try: validate(result.data, self.schema) except Exception as e: diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py index 60838a1d..86d21d57 100644 --- a/src/scanoss/utils/file.py +++ b/src/scanoss/utils/file.py @@ -21,18 +21,22 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + import json import os -import sys from dataclasses import dataclass from typing import Optional +JSON_ERROR_PARSE = 1 +JSON_ERROR_FILE_NOT_FOUND = 2 + @dataclass class JsonValidation: is_valid: bool data: Optional[dict] = None error: Optional[str] = None + error_code: Optional[int] = None def validate_json_file(json_file_path: str) -> JsonValidation: @@ -46,12 +50,20 @@ def validate_json_file(json_file_path: str) -> JsonValidation: Tuple[bool, str]: A tuple containing a boolean indicating if the file is valid and a message """ if not json_file_path: - return JsonValidation(is_valid=False, error='No JSON file specified') + return JsonValidation(is_valid=False, error="No JSON file specified") if not os.path.isfile(json_file_path): - return JsonValidation(is_valid=False, error=f'File not found: {json_file_path}') + return JsonValidation( + is_valid=False, + error=f"File not found: {json_file_path}", + error_code=JSON_ERROR_FILE_NOT_FOUND, + ) try: with open(json_file_path) as f: data = json.load(f) return JsonValidation(is_valid=True, data=data) except json.JSONDecodeError as e: - return JsonValidation(is_valid=False, error=f'Problem parsing JSON file: "{json_file_path}": {e}') + return JsonValidation( + is_valid=False, + error=f'Problem parsing JSON file: "{json_file_path}": {e}', + error_code=JSON_ERROR_PARSE, + ) From 7f086430721c4f786ace98c56470200e431c3bec Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 30 Jan 2025 13:38:45 +0100 Subject: [PATCH 266/489] feat: ES-148 Look scan settings file inside the folder being scanned instead of cwd --- CHANGELOG.md | 1 + src/scanoss/cli.py | 6 +++--- src/scanoss/scanoss_settings.py | 16 ++++++++++------ src/scanoss/utils/file.py | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c901c66..da5ed86c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.19.6] - 2025-01-30 ### Added - Omit settings file if it does not exist instead of throwing an error. +- Look settings file inside the folder being scanned instead of the cwd. ## [1.19.5] - 2025-01-14 ### Added diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index fa3b583b..4121ae42 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -575,11 +575,11 @@ def scan(parser, args): scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) try: if args.identify: - scan_settings.load_json_file(args.identify).set_file_type('legacy').set_scan_type('identify') + scan_settings.load_json_file(args.identify, args.scan_dir).set_file_type('legacy').set_scan_type('identify') elif args.ignore: - scan_settings.load_json_file(args.ignore).set_file_type('legacy').set_scan_type('blacklist') + scan_settings.load_json_file(args.ignore, args.scan_dir).set_file_type('legacy').set_scan_type('blacklist') else: - scan_settings.load_json_file(args.settings).set_file_type('new').set_scan_type('identify') + scan_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new').set_scan_type('identify') except ScanossSettingsError as e: print_stderr(f'Error: {e}') exit(1) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index e5819a76..cec4449a 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -30,9 +30,9 @@ from jsonschema import validate from .scanossbase import ScanossBase -from .utils.file import JSON_ERROR_FILE_NOT_FOUND, validate_json_file +from .utils.file import JSON_ERROR_FILE_NOT_FOUND, JSON_ERROR_FILE_EMPTY, validate_json_file -DEFAULT_SCANOSS_JSON_FILE = 'scanoss.json' +DEFAULT_SCANOSS_JSON_FILE = Path('scanoss.json') class BomEntry(TypedDict, total=False): @@ -96,16 +96,20 @@ def __init__( if filepath: self.load_json_file(filepath) - def load_json_file(self, filepath: 'str | None' = None) -> 'ScanossSettings': + def load_json_file(self, filepath: 'str | None' = None, scan_root: 'str | None' = None) -> 'ScanossSettings': """ Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. Args: filepath (str): Path to the SCANOSS settings file """ + if not filepath: filepath = DEFAULT_SCANOSS_JSON_FILE - json_file = Path(filepath).resolve() + + filepath = Path(scan_root) / filepath if scan_root else Path(filepath) + + json_file = filepath.resolve() if filepath == DEFAULT_SCANOSS_JSON_FILE and not json_file.exists(): self.print_debug(f'Default settings file "{filepath}" not found. Skipping...') @@ -114,8 +118,8 @@ def load_json_file(self, filepath: 'str | None' = None) -> 'ScanossSettings': result = validate_json_file(json_file) if not result.is_valid: - if result.error_code == JSON_ERROR_FILE_NOT_FOUND: - self.print_debug(f'The provided settings file "{filepath}" was not found. Skipping...') + if result.error_code == JSON_ERROR_FILE_NOT_FOUND or result.error_code == JSON_ERROR_FILE_EMPTY: + self.print_msg(f'WARNING: The supplied settings file "{filepath}" was not found or is empty. Skipping...') return self else: raise ScanossSettingsError(f'Problem with settings file. {result.error}') diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py index 86d21d57..e64d7d7a 100644 --- a/src/scanoss/utils/file.py +++ b/src/scanoss/utils/file.py @@ -29,6 +29,8 @@ JSON_ERROR_PARSE = 1 JSON_ERROR_FILE_NOT_FOUND = 2 +JSON_ERROR_FILE_EMPTY = 3 +JSON_ERROR_FILE_SIZE = 4 @dataclass @@ -57,6 +59,19 @@ def validate_json_file(json_file_path: str) -> JsonValidation: error=f"File not found: {json_file_path}", error_code=JSON_ERROR_FILE_NOT_FOUND, ) + try: + if os.stat(json_file_path).st_size == 0: + return JsonValidation( + is_valid=False, + error=f"File is empty: {json_file_path}", + error_code=JSON_ERROR_FILE_EMPTY, + ) + except OSError as e: + return JsonValidation( + is_valid=False, + error=f"Problem checking file size: {json_file_path}: {e}", + error_code=JSON_ERROR_FILE_SIZE, + ) try: with open(json_file_path) as f: data = json.load(f) From 2492cf9cdd743e832daab6f36c5ae0abc968c9f6 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Wed, 18 Sep 2024 11:23:20 -0300 Subject: [PATCH 267/489] feat: SP-1523 Add provenance decoration --- CHANGELOG.md | 8 +- CLIENT_HELP.md | 10 ++ src/scanoss/api/provenance/__init__.py | 23 ++++ src/scanoss/api/provenance/v2/__init__.py | 23 ++++ .../provenance/v2/scanoss_provenance_pb2.py | 42 +++++++ .../v2/scanoss_provenance_pb2_grpc.py | 108 ++++++++++++++++++ src/scanoss/cli.py | 43 ++++++- src/scanoss/components.py | 26 +++++ src/scanoss/scanossgrpc.py | 33 ++++++ tests/grpc-client-test.py | 14 +++ 10 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 src/scanoss/api/provenance/__init__.py create mode 100644 src/scanoss/api/provenance/v2/__init__.py create mode 100644 src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py create mode 100644 src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index da5ed86c..3cd450b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.0] - 2025-02-02 +### Added +- Added support for component provenance reporting + - `scanoss-py component prov ...` + ## [1.19.6] - 2025-01-30 ### Added - Omit settings file if it does not exist instead of throwing an error. @@ -450,4 +455,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.19.2]: https://github.com/scanoss/scanoss.py/compare/v1.19.1...v1.19.2 [1.19.3]: https://github.com/scanoss/scanoss.py/compare/v1.19.2...v1.19.3 [1.19.4]: https://github.com/scanoss/scanoss.py/compare/v1.19.3...v1.19.4 -[1.19.5]: https://github.com/scanoss/scanoss.py/compare/v1.19.4...v1.19.5 \ No newline at end of file +[1.19.5]: https://github.com/scanoss/scanoss.py/compare/v1.19.4...v1.19.5 +[1.20.0]: https://github.com/scanoss/scanoss.py/compare/v1.19.5...v1.20.0 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 3f0cdbb0..bf54fa36 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -298,6 +298,16 @@ scanoss-py comp semgrep --key $SC_API_KEY -i purl-input.json -o semgrep-issues.j ``` **Note:** This sub-command requires a subscription to SCANOSS premium data. +#### Component Provenance +The following command provides the capability to search the SCANOSS KB for component Provenance: +```bash +scanoss-py comp prov -p "pkg:github/unoconv/unoconv" +``` +It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): +```bash +scanoss-py comp prov -i purl-input.json -o vulnernable-comps.json + + ### Results Commands The `results` command provides the capability to operate on scan results. For example: diff --git a/src/scanoss/api/provenance/__init__.py b/src/scanoss/api/provenance/__init__.py new file mode 100644 index 00000000..d2ed05b0 --- /dev/null +++ b/src/scanoss/api/provenance/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/provenance/v2/__init__.py b/src/scanoss/api/provenance/v2/__init__.py new file mode 100644 index 00000000..d2ed05b0 --- /dev/null +++ b/src/scanoss/api/provenance/v2/__init__.py @@ -0,0 +1,23 @@ +""" + SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py new file mode 100644 index 00000000..3bbb92b8 --- /dev/null +++ b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: scanoss/api/provenance/v2/scanoss-provenance.proto +# Protobuf Python Version: 4.25.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/provenance/v2/scanoss-provenance.proto\x12\x19scanoss.api.provenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xd5\x03\n\x12ProvenanceResponse\x12\x42\n\x05purls\x18\x01 \x03(\x0b\x32\x33.scanoss.api.provenance.v2.ProvenanceResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xdc\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12Z\n\x12\x64\x65\x63lared_locations\x18\x03 \x03(\x0b\x32>.scanoss.api.provenance.v2.ProvenanceResponse.DeclaredLocation\x12X\n\x11\x63urated_locations\x18\x04 \x03(\x0b\x32=.scanoss.api.provenance.v2.ProvenanceResponse.CuratedLocation2\x98\x02\n\nProvenance\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/provenance/echo:\x01*\x12\x94\x01\n\x16GetComponentProvenance\x12\".scanoss.api.common.v2.PurlRequest\x1a-.scanoss.api.provenance.v2.ProvenanceResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/provenance/countries:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/provenancev2;provenancev2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Provenance Service\"P\n\x12scanoss-provenance\x12%https://github.com/scanoss/provenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.provenance.v2.scanoss_provenance_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/provenancev2;provenancev2\222A\331\001\022s\n\032SCANOSS Provenance Service\"P\n\022scanoss-provenance\022%https://github.com/scanoss/provenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_PROVENANCE'].methods_by_name['Echo']._options = None + _globals['_PROVENANCE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/provenance/echo:\001*' + _globals['_PROVENANCE'].methods_by_name['GetComponentProvenance']._options = None + _globals['_PROVENANCE'].methods_by_name['GetComponentProvenance']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/provenance/countries:\001*' + _globals['_PROVENANCERESPONSE']._serialized_start=202 + _globals['_PROVENANCERESPONSE']._serialized_end=671 + _globals['_PROVENANCERESPONSE_DECLAREDLOCATION']._serialized_start=347 + _globals['_PROVENANCERESPONSE_DECLAREDLOCATION']._serialized_end=397 + _globals['_PROVENANCERESPONSE_CURATEDLOCATION']._serialized_start=399 + _globals['_PROVENANCERESPONSE_CURATEDLOCATION']._serialized_end=448 + _globals['_PROVENANCERESPONSE_PURLS']._serialized_start=451 + _globals['_PROVENANCERESPONSE_PURLS']._serialized_end=671 + _globals['_PROVENANCE']._serialized_start=674 + _globals['_PROVENANCE']._serialized_end=954 +# @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py new file mode 100644 index 00000000..7143ca60 --- /dev/null +++ b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py @@ -0,0 +1,108 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from scanoss.api.provenance.v2 import scanoss_provenance_pb2 as scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2 + + +class ProvenanceStub(object): + """* + Expose all of the SCANOSS Provenance RPCs here + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Echo = channel.unary_unary( + '/scanoss.api.provenance.v2.Provenance/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) + self.GetComponentProvenance = channel.unary_unary( + '/scanoss.api.provenance.v2.Provenance/GetComponentProvenance', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2.ProvenanceResponse.FromString, + ) + + +class ProvenanceServicer(object): + """* + Expose all of the SCANOSS Provenance RPCs here + """ + + def Echo(self, request, context): + """Standard echo + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentProvenance(self, request, context): + """Get Provenance countrues associated with a list of PURLs + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ProvenanceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetComponentProvenance': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentProvenance, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2.ProvenanceResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.provenance.v2.Provenance', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Provenance(object): + """* + Expose all of the SCANOSS Provenance RPCs here + """ + + @staticmethod + def Echo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.Provenance/Echo', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetComponentProvenance(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.Provenance/GetComponentProvenance', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2.ProvenanceResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 4121ae42..ce61e9bf 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -198,6 +198,13 @@ def setup_args() -> None: help='Retrieve semgrep issues/findings for the given components') c_semgrep.set_defaults(func=comp_semgrep) + # Component Sub-command: component semgrep + c_provenance = comp_sub.add_parser('provenance', aliases=['prov'], + description=f'Show Provenance findings: {__version__}', + help='Retrieve provenance for the given components') + c_provenance.set_defaults(func=comp_provenance) + + # Component Sub-command: component search c_search = comp_sub.add_parser('search', aliases=['sc'], description=f'Search component details: {__version__}', @@ -221,11 +228,11 @@ def setup_args() -> None: c_versions.set_defaults(func=comp_versions) # Common purl Component sub-command options - for p in [c_crypto, c_vulns, c_semgrep]: + for p in [c_crypto, c_vulns, c_semgrep, c_provenance]: p.add_argument('--purl', '-p', type=str, nargs="*", help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') # Common Component sub-command options - for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep]: + for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p.add_argument('--timeout', '-M', type=int, default=600, help='Timeout (in seconds) for API communication (optional - default 600)') @@ -361,7 +368,7 @@ def setup_args() -> None: p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.') # Global Scan/GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: p.add_argument('--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)') p.add_argument('--proxy', type=str, help='Proxy URL to use for connections (optional). ' @@ -375,7 +382,7 @@ def setup_args() -> None: '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC') # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: p.add_argument('--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)') p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' @@ -383,7 +390,7 @@ def setup_args() -> None: # Help/Trace command options for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto, c_vulns, c_search, - c_versions, c_semgrep, p_results, p_undeclared, p_copyleft]: + c_versions, c_semgrep, p_results, p_undeclared, p_copyleft, c_provenance]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -473,7 +480,7 @@ def wfp(parser, args): if args.output: scan_output = args.output open(scan_output, 'w').close() - + # Load scan settings scan_settings = None if not args.skip_settings_file: @@ -1022,6 +1029,30 @@ def comp_semgrep(parser, args): if not comps.get_semgrep_details(args.input, args.purl, args.output): exit(1) +def comp_provenance(parser, args): + """ + Run the "component semgrep" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + exit(1) + pac_file = get_pac_file(args.pac) + comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + timeout=args.timeout) + if not comps.get_provenance_details(args.input, args.purl, args.output): + exit(1) + def comp_search(parser, args): """ Run the "component search" sub-command diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 705bf04c..eb6f70ae 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -302,3 +302,29 @@ def get_component_versions(self, output_file: str = None, json_file: str = None, self.print_msg(f'Results written to: {output_file}') self._close_file(output_file, file) return success + + def get_provenance_details(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: + """ + Retrieve the semgrep details for the supplied PURLs + + :param json_file: PURL JSON request file (optional) + :param purls: PURL request array (optional) + :param output_file: output filename (optional). Default: STDOUT + :return: True on success, False otherwise + """ + success = False + purls_request = self.load_purls(json_file, purls) + if purls_request is None or len(purls_request) == 0: + return False + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + self.print_msg('Sending PURLs to Provenance API for decoration...') + response = self.grpc_api.get_provenance_json(purls_request) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success \ No newline at end of file diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 80bec813..5519bff3 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -37,6 +37,7 @@ from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub +from .api.provenance.v2.scanoss_provenance_pb2_grpc import ProvenanceStub from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, DependencyResponse @@ -45,6 +46,8 @@ from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse from .api.components.v2.scanoss_components_pb2 import (CompSearchRequest, CompSearchResponse, CompVersionRequest, CompVersionResponse) +from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse + from .scanossbase import ScanossBase from . import __version__ @@ -113,6 +116,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) self.semgrep_stub = SemgrepStub(grpc.insecure_channel(self.url)) self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url)) + self.provenance_stub = ProvenanceStub(grpc.insecure_channel(self.url)) else: if ca_cert is not None: credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate @@ -123,6 +127,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) self.semgrep_stub = SemgrepStub(grpc.secure_channel(self.url, credentials)) self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials)) + self.provenance_stub = ProvenanceStub(grpc.secure_channel(self.url, credentials)) @classmethod def _load_cert(cls, cert_file: str) -> bytes: @@ -414,6 +419,34 @@ def _get_proxy_config(self): os.environ["http_proxy"] = proxies.get("http") or "" os.environ["https_proxy"] = proxies.get("https") or "" + def get_provenance_json(self, purls: dict) -> dict: + """ + Client function to call the rpc for GetComponentProvenance + :param purls: Message to send to the service + :return: Server response or None + """ + if not purls: + self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + return None + request_id = str(uuid.uuid4()) + resp: ProvenanceResponse + try: + request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) # Set a Request ID + self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...') + resp = self.provenance_stub.GetComponentProvenance(request, metadata=metadata, timeout=self.timeout) + except Exception as e: + self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' + f'(rqId: {request_id}): {e}') + else: + if resp: + if not self._check_status_response(resp.status, request_id): + return None + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict + del resp_dict['status'] + return resp_dict + return None # # End of ScanossGrpc Class # diff --git a/tests/grpc-client-test.py b/tests/grpc-client-test.py index d9b2c609..4630bed0 100644 --- a/tests/grpc-client-test.py +++ b/tests/grpc-client-test.py @@ -24,7 +24,9 @@ import os import unittest +from math import trunc +from scanoss.components import Components from scanoss.scancodedeps import ScancodeDeps from scanoss.scanossgrpc import ScanossGrpc @@ -75,6 +77,18 @@ def test_grpc_get_dependencies(self): file = dep_file.pop('file', None) print(f'File: {file} - {dep_file}') + def test_load_purls_array(self): + comps = Components(debug=True, trace=True) + # Expected value as a dictionary, not a string + expected_value = { + 'purls': [ + {'purl': 'pkg:github/unoconv/unoconv'}, + {'purl': 'pkg:github/torvalds/linux@v5.13'} + ] + } + components = comps.load_purls(purls=["pkg:github/unoconv/unoconv", "pkg:github/torvalds/linux@v5.13"]) + print(components) + self.assertEqual(components,expected_value) if __name__ == '__main__': unittest.main() From 151727f06f6b98f82fd69ac497ac431bf70b18b6 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Thu, 19 Sep 2024 07:55:11 -0300 Subject: [PATCH 268/489] chore: SP-1531 Adds unit tests --- CLIENT_HELP.md | 2 +- src/scanoss/cli.py | 2 +- src/scanoss/components.py | 5 ++++- src/scanoss/scanossgrpc.py | 1 - tests/grpc-client-test.py | 34 +++++++++++++++++++++++++++++++++- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index bf54fa36..8eb0180a 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -305,7 +305,7 @@ scanoss-py comp prov -p "pkg:github/unoconv/unoconv" ``` It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): ```bash -scanoss-py comp prov -i purl-input.json -o vulnernable-comps.json +scanoss-py comp prov -i purl-input.json -o provenance.json ### Results Commands diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ce61e9bf..5615737a 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1031,7 +1031,7 @@ def comp_semgrep(parser, args): def comp_provenance(parser, args): """ - Run the "component semgrep" sub-command + Run the "component provenance" sub-command Parameters ---------- parser: ArgumentParser diff --git a/src/scanoss/components.py b/src/scanoss/components.py index eb6f70ae..28f6fd66 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -62,7 +62,7 @@ def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False ver_details=ver_details, ca_cert=ca_cert, proxy=proxy, pac=pac, grpc_proxy=grpc_proxy, timeout=timeout) - def load_purls(self, json_file: str = None, purls: [] = None) -> dict: + def load_purls(self, json_file: str = None, purls: [str] = None) -> dict: """ Load the specified purls and return a dictionary @@ -81,6 +81,9 @@ def load_purls(self, json_file: str = None, purls: [] = None) -> dict: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return None elif purls: + if not all(isinstance(purl, str) for purl in purls): + self.print_stderr('ERROR: PURLs must be a list of strings.') + return None parsed_purls = [] for p in purls: parsed_purls.append({'purl': p}) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 5519bff3..b5d8f7df 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -444,7 +444,6 @@ def get_provenance_json(self, purls: dict) -> dict: if not self._check_status_response(resp.status, request_id): return None resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict - del resp_dict['status'] return resp_dict return None # diff --git a/tests/grpc-client-test.py b/tests/grpc-client-test.py index 4630bed0..2333501e 100644 --- a/tests/grpc-client-test.py +++ b/tests/grpc-client-test.py @@ -24,7 +24,9 @@ import os import unittest -from math import trunc +from unittest.mock import patch +from io import StringIO + from scanoss.components import Components from scanoss.scancodedeps import ScancodeDeps @@ -87,7 +89,37 @@ def test_load_purls_array(self): ] } components = comps.load_purls(purls=["pkg:github/unoconv/unoconv", "pkg:github/torvalds/linux@v5.13"]) + self.assertEqual(components,expected_value) + + @patch('sys.stderr', new_callable=StringIO) + def test_load_purls_array_malformed(self, mock_stderr): + comps = Components(debug=True, trace=True) + components = comps.load_purls(purls=[1, "pkg:github/torvalds/linux@v5.13"]) + print(components) + self.assertEqual(components,None) + self.assertIn('ERROR: PURLs must be a list of strings.', mock_stderr.getvalue()) + + @patch('sys.stderr', new_callable=StringIO) + def test_load_purls_file_malformed(self, mock_stderr): + comps = Components(debug=True, trace=True) + components = comps.load_purls(json_file='./data/malformed-purl-input.json') + # Ensure the method returned None (indicating a failure) + self.assertIsNone(components) + # Check if the correct error message was printed to stderr + self.assertIn('ERROR: No PURLs parsed from request.', mock_stderr.getvalue()) + + def test_load_purls_file(self): + comps = Components(debug=True, trace=True) + expected_value = { + 'purls': [ + { + 'purl': 'pkg:github/torvalds/linux@v5.13' + } + ] + } + components = comps.load_purls( json_file='./data/purl-input.json') print(components) + # Ensure the method returned None (indicating a failure) self.assertEqual(components,expected_value) if __name__ == '__main__': From 84ebdab495fba03d5fefef17451064b64da55a46 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Thu, 19 Sep 2024 09:40:34 -0300 Subject: [PATCH 269/489] chore: Upgrades version to v1.20.0 --- src/scanoss/__init__.py | 2 +- src/scanoss/components.py | 6 +++--- tests/grpc-client-test.py | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 758d61eb..f691244c 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = "1.19.6" +__version__ = "1.20.0" diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 28f6fd66..37ebfefa 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -24,7 +24,7 @@ import json import os import sys -from typing import TextIO +from typing import TextIO, Optional, List from pypac.parser import PACFile @@ -62,13 +62,13 @@ def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False ver_details=ver_details, ca_cert=ca_cert, proxy=proxy, pac=pac, grpc_proxy=grpc_proxy, timeout=timeout) - def load_purls(self, json_file: str = None, purls: [str] = None) -> dict: + def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None) -> Optional[dict]: """ Load the specified purls and return a dictionary :param json_file: JSON PURL file (optional) :param purls: list of PURLs (optional) - :return: PURL Request dictionary + :return: PURL Request dictionary or None """ if json_file: if not os.path.isfile(json_file) or not os.access(json_file, os.R_OK): diff --git a/tests/grpc-client-test.py b/tests/grpc-client-test.py index 2333501e..57dee58b 100644 --- a/tests/grpc-client-test.py +++ b/tests/grpc-client-test.py @@ -95,7 +95,6 @@ def test_load_purls_array(self): def test_load_purls_array_malformed(self, mock_stderr): comps = Components(debug=True, trace=True) components = comps.load_purls(purls=[1, "pkg:github/torvalds/linux@v5.13"]) - print(components) self.assertEqual(components,None) self.assertIn('ERROR: PURLs must be a list of strings.', mock_stderr.getvalue()) From 762709b78db4239eb4a62df9a1b63a095d946957 Mon Sep 17 00:00:00 2001 From: agusgroh Date: Thu, 19 Sep 2024 11:07:19 -0300 Subject: [PATCH 270/489] chore:ER-109 Adds provenance alias on cli command --- CLIENT_HELP.md | 1 + src/scanoss/cli.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 8eb0180a..6837fafa 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -240,6 +240,7 @@ The `component` command has a suite of sub-commands designed to operate on OSS c * Search (`search`) * Version Details (`versions`) * Cryptography (`crypto`) +* Provenance (`provenance`) For the latest list of sub-commands, please run: ```bash diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 5615737a..c2e36a56 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -198,8 +198,8 @@ def setup_args() -> None: help='Retrieve semgrep issues/findings for the given components') c_semgrep.set_defaults(func=comp_semgrep) - # Component Sub-command: component semgrep - c_provenance = comp_sub.add_parser('provenance', aliases=['prov'], + # Component Sub-command: component provenance + c_provenance = comp_sub.add_parser('provenance', aliases=['prov', 'prv'], description=f'Show Provenance findings: {__version__}', help='Retrieve provenance for the given components') c_provenance.set_defaults(func=comp_provenance) From 1906c4268b494dcd5506e78bded63101f42d4716 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Wed, 5 Feb 2025 13:10:09 +0000 Subject: [PATCH 271/489] chore:ER-108 gRPC error check for succeed with warnings --- src/scanoss/scanossgrpc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index b5d8f7df..b8475233 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -388,13 +388,15 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st self.print_debug(f'Checking response status (rqId: {request_id}): {status_response}') status_code: StatusCode = status_response.status if status_code > 1: + ret_val = False # default to failed msg = "Unsuccessful" if status_code == 2: msg = "Succeeded with warnings" + ret_val = True # No need to fail as it succeeded with warnings elif status_code == 3: msg = "Failed with warnings" self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}') - return False + return ret_val return True def _get_proxy_config(self): From b3ec0cf5d524cbf09698fffdf647d269cf239e32 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 4 Feb 2025 11:54:07 +0100 Subject: [PATCH 272/489] chore: add ruff as formater and linter, add pre commit hooks, add lint workflow chore: update python version on lint action chore: update lint action chore: add ruff as formater and linter, add pre commit hooks, add lint workflow --- .github/workflows/lint.yml | 52 ++ .pre-commit-config.yaml | 6 + date_time.py | 37 +- docs/source/conf.py | 3 +- pyproject.toml | 17 +- requirements-dev.txt | 2 + src/protoc_gen_swagger/__init__.py | 26 +- src/protoc_gen_swagger/options/__init__.py | 26 +- .../options/annotations_pb2.py | 21 +- .../options/annotations_pb2_grpc.py | 2 +- .../options/openapiv2_pb2.py | 194 ++--- .../options/openapiv2_pb2_grpc.py | 2 +- src/scanoss/__init__.py | 36 +- src/scanoss/api/__init__.py | 34 +- src/scanoss/api/common/__init__.py | 34 +- src/scanoss/api/common/v2/__init__.py | 34 +- .../api/common/v2/scanoss_common_pb2.py | 36 +- .../api/common/v2/scanoss_common_pb2_grpc.py | 2 +- src/scanoss/api/components/__init__.py | 34 +- src/scanoss/api/components/v2/__init__.py | 34 +- .../components/v2/scanoss_components_pb2.py | 86 +- .../v2/scanoss_components_pb2_grpc.py | 238 ++--- .../v2/scanoss_cryptography_pb2.py | 38 +- .../v2/scanoss_cryptography_pb2_grpc.py | 124 +-- src/scanoss/api/dependencies/__init__.py | 34 +- src/scanoss/api/dependencies/v2/__init__.py | 34 +- .../v2/scanoss_dependencies_pb2.py | 54 +- .../v2/scanoss_dependencies_pb2_grpc.py | 124 +-- src/scanoss/api/scanning/__init__.py | 34 +- src/scanoss/api/scanning/v2/__init__.py | 34 +- .../api/scanning/v2/scanoss_scanning_pb2.py | 18 +- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 72 +- src/scanoss/api/semgrep/__init__.py | 34 +- src/scanoss/api/semgrep/v2/__init__.py | 34 +- .../api/semgrep/v2/scanoss_semgrep_pb2.py | 40 +- .../semgrep/v2/scanoss_semgrep_pb2_grpc.py | 120 +-- src/scanoss/api/vulnerabilities/__init__.py | 34 +- .../api/vulnerabilities/v2/__init__.py | 34 +- .../v2/scanoss_vulnerabilities_pb2.py | 64 +- .../v2/scanoss_vulnerabilities_pb2_grpc.py | 181 ++-- src/scanoss/cli.py | 810 ++++++++++++------ src/scanoss/components.py | 112 ++- src/scanoss/csvoutput.py | 139 +-- src/scanoss/cyclonedx.py | 94 +- src/scanoss/file_filters.py | 28 +- src/scanoss/filecount.py | 79 +- src/scanoss/inspection/__init__.py | 34 +- src/scanoss/inspection/copyleft.py | 129 +-- src/scanoss/inspection/policy_check.py | 129 +-- .../inspection/undeclared_component.py | 173 ++-- src/scanoss/inspection/utils/license_utils.py | 110 ++- src/scanoss/results.py | 111 ++- src/scanoss/scancodedeps.py | 99 ++- src/scanoss/scanner.py | 338 +++++--- src/scanoss/scanoss_settings.py | 8 +- src/scanoss/scanossapi.py | 167 ++-- src/scanoss/scanossbase.py | 38 +- src/scanoss/scanossgrpc.py | 124 +-- src/scanoss/scanpostprocessor.py | 15 +- src/scanoss/scantype.py | 43 +- src/scanoss/spdxlite.py | 103 +-- src/scanoss/threadeddependencies.py | 152 ++-- src/scanoss/threadedscanning.py | 68 +- src/scanoss/utils/file.py | 8 +- src/scanoss/winnowing.py | 158 +++- tests/test_csv_output.py | 50 +- tests/test_file_filters.py | 81 +- tests/test_policy_inspect.py | 160 ++-- tests/test_winnowing.py | 63 +- version.py | 47 +- 70 files changed, 3354 insertions(+), 2379 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..a110d651 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,52 @@ +name: Lint + +on: + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + + - name: Get changed Python files + id: changed_files + run: | + # Find the merge base between the main branch and the current HEAD. + merge_base=$(git merge-base origin/main HEAD) + # List all changed Python files since the merge base. + files=$(git diff --name-only "$merge_base" HEAD | grep '\.py$' || true) + + # Use the multi-line syntax for outputs. + echo "files<> "$GITHUB_OUTPUT" + echo "${files}" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + echo "Changed files: ${files}" + + - name: Run Ruff on changed files + run: | + if [ -z "${{ steps.changed_files.outputs.files }}" ]; then + echo "No Python files changed. Exiting." + exit 0 + else + echo "Linting the following files:" + echo "${{ steps.changed_files.outputs.files }}" + # Pass the list of changed files to Ruff. + echo "${{ steps.changed_files.outputs.files }}" | xargs ruff check + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..bb2e62f0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.4 + hooks: + - id: ruff + - id: ruff-format diff --git a/date_time.py b/date_time.py index 99280f5a..791d3d34 100755 --- a/date_time.py +++ b/date_time.py @@ -1,33 +1,34 @@ #!/usr/bin/env python3 """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2022, SCANOSS + Copyright (c) 2022, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import datetime """ Store the current date/time into a data field for processing """ -if __name__ == "__main__": +if __name__ == '__main__': now = datetime.datetime.now() data = f'date: {now.strftime("%Y%m%d%H%M%S")}, utime: {int(now.timestamp())}' with open('src/scanoss/data/build_date.txt', 'w') as f: diff --git a/docs/source/conf.py b/docs/source/conf.py index 19fd7e9a..4372835b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,7 @@ extensions = ['sphinx_rtd_theme'] -templates_path = ["_templates"] +templates_path = ['_templates'] exclude_patterns = [] @@ -27,4 +27,3 @@ html_theme = 'sphinx_rtd_theme' html_logo = 'scanosslogo.jpg' html_static_path = ['_static'] - diff --git a/pyproject.toml b/pyproject.toml index 48fb09ac..29a06a43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,18 @@ [build-system] requires = ["setuptools", "wheel", "twine"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +[tool.ruff] +# Enable pycodestyle (E), pyflakes (F), isort (I), pylint (PL) +select = ["E", "F", "I", "PL"] +line-length = 120 +# Assume Python 3.7+ +target-version = "py37" + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" +line-ending = "auto" + +[tool.ruff.lint.isort] +known-first-party = ["scanoss"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 0187cb26..083ebdf2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,5 @@ wheel twine build grpcio-tools +ruff +pre-commit \ No newline at end of file diff --git a/src/protoc_gen_swagger/__init__.py b/src/protoc_gen_swagger/__init__.py index 6b78cbea..6f3297a5 100644 --- a/src/protoc_gen_swagger/__init__.py +++ b/src/protoc_gen_swagger/__init__.py @@ -1,20 +1,20 @@ """ - SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause - Copyright (c) 2015, Gengo, Inc. - All rights reserved. + Copyright (c) 2015, Gengo, Inc. + All rights reserved. - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. - * Neither the name of Gengo, Inc. nor the names of its - contributors may be used to endorse or promote products derived from this - software without specific prior written permission. + * Neither the name of Gengo, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. """ diff --git a/src/protoc_gen_swagger/options/__init__.py b/src/protoc_gen_swagger/options/__init__.py index 6b78cbea..6f3297a5 100644 --- a/src/protoc_gen_swagger/options/__init__.py +++ b/src/protoc_gen_swagger/options/__init__.py @@ -1,20 +1,20 @@ """ - SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause - Copyright (c) 2015, Gengo, Inc. - All rights reserved. + Copyright (c) 2015, Gengo, Inc. + All rights reserved. - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. - * Neither the name of Gengo, Inc. nor the names of its - contributors may be used to endorse or promote products derived from this - software without specific prior written permission. + * Neither the name of Gengo, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. """ diff --git a/src/protoc_gen_swagger/options/annotations_pb2.py b/src/protoc_gen_swagger/options/annotations_pb2.py index c568f388..58f85f85 100644 --- a/src/protoc_gen_swagger/options/annotations_pb2.py +++ b/src/protoc_gen_swagger/options/annotations_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: protoc-gen-swagger/options/annotations.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -15,17 +16,19 @@ from protoc_gen_swagger.options import openapiv2_pb2 as protoc__gen__swagger_dot_options_dot_openapiv2__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,protoc-gen-swagger/options/annotations.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a google/protobuf/descriptor.proto\x1a*protoc-gen-swagger/options/openapiv2.proto:j\n\x11openapiv2_swagger\x12\x1c.google.protobuf.FileOptions\x18\x92\x08 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Swagger:p\n\x13openapiv2_operation\x12\x1e.google.protobuf.MethodOptions\x18\x92\x08 \x01(\x0b\x32\x32.grpc.gateway.protoc_gen_swagger.options.Operation:k\n\x10openapiv2_schema\x12\x1f.google.protobuf.MessageOptions\x18\x92\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema:e\n\ropenapiv2_tag\x12\x1f.google.protobuf.ServiceOptions\x18\x92\x08 \x01(\x0b\x32,.grpc.gateway.protoc_gen_swagger.options.Tag:l\n\x0fopenapiv2_field\x12\x1d.google.protobuf.FieldOptions\x18\x92\x08 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchemaBCZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b"\n,protoc-gen-swagger/options/annotations.proto\x12'grpc.gateway.protoc_gen_swagger.options\x1a google/protobuf/descriptor.proto\x1a*protoc-gen-swagger/options/openapiv2.proto:j\n\x11openapiv2_swagger\x12\x1c.google.protobuf.FileOptions\x18\x92\x08 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Swagger:p\n\x13openapiv2_operation\x12\x1e.google.protobuf.MethodOptions\x18\x92\x08 \x01(\x0b\x32\x32.grpc.gateway.protoc_gen_swagger.options.Operation:k\n\x10openapiv2_schema\x12\x1f.google.protobuf.MessageOptions\x18\x92\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema:e\n\ropenapiv2_tag\x12\x1f.google.protobuf.ServiceOptions\x18\x92\x08 \x01(\x0b\x32,.grpc.gateway.protoc_gen_swagger.options.Tag:l\n\x0fopenapiv2_field\x12\x1d.google.protobuf.FieldOptions\x18\x92\x08 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchemaBCZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3" +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.annotations_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - google_dot_protobuf_dot_descriptor__pb2.FileOptions.RegisterExtension(openapiv2_swagger) - google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(openapiv2_operation) - google_dot_protobuf_dot_descriptor__pb2.MessageOptions.RegisterExtension(openapiv2_schema) - google_dot_protobuf_dot_descriptor__pb2.ServiceOptions.RegisterExtension(openapiv2_tag) - google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(openapiv2_field) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' + google_dot_protobuf_dot_descriptor__pb2.FileOptions.RegisterExtension(openapiv2_swagger) + google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(openapiv2_operation) + google_dot_protobuf_dot_descriptor__pb2.MessageOptions.RegisterExtension(openapiv2_schema) + google_dot_protobuf_dot_descriptor__pb2.ServiceOptions.RegisterExtension(openapiv2_tag) + google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(openapiv2_field) + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' # @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/annotations_pb2_grpc.py b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py index 2daafffe..bf947056 100644 --- a/src/protoc_gen_swagger/options/annotations_pb2_grpc.py +++ b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py @@ -1,4 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" -import grpc +import grpc diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2.py b/src/protoc_gen_swagger/options/openapiv2_pb2.py index 0df96e43..5c01c38b 100644 --- a/src/protoc_gen_swagger/options/openapiv2_pb2.py +++ b/src/protoc_gen_swagger/options/openapiv2_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: protoc-gen-swagger/options/openapiv2.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -15,104 +16,105 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*protoc-gen-swagger/options/openapiv2.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xa0\x07\n\x07Swagger\x12\x0f\n\x07swagger\x18\x01 \x01(\t\x12;\n\x04info\x18\x02 \x01(\x0b\x32-.grpc.gateway.protoc_gen_swagger.options.Info\x12\x0c\n\x04host\x18\x03 \x01(\t\x12\x11\n\tbase_path\x18\x04 \x01(\t\x12O\n\x07schemes\x18\x05 \x03(\x0e\x32>.grpc.gateway.protoc_gen_swagger.options.Swagger.SwaggerScheme\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12R\n\tresponses\x18\n \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Swagger.ResponsesEntry\x12Z\n\x14security_definitions\x18\x0b \x01(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12U\n\rexternal_docs\x18\x0e \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12T\n\nextensions\x18\x0f \x03(\x0b\x32@.grpc.gateway.protoc_gen_swagger.options.Swagger.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"B\n\rSwaggerScheme\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04HTTP\x10\x01\x12\t\n\x05HTTPS\x10\x02\x12\x06\n\x02WS\x10\x03\x12\x07\n\x03WSS\x10\x04J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\r\x10\x0e\"\xa9\x05\n\tOperation\x12\x0c\n\x04tags\x18\x01 \x03(\t\x12\x0f\n\x07summary\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12U\n\rexternal_docs\x18\x04 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12\x14\n\x0coperation_id\x18\x05 \x01(\t\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12T\n\tresponses\x18\t \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Operation.ResponsesEntry\x12\x0f\n\x07schemes\x18\n \x03(\t\x12\x12\n\ndeprecated\x18\x0b \x01(\x08\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12V\n\nextensions\x18\r \x03(\x0b\x32\x42.grpc.gateway.protoc_gen_swagger.options.Operation.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01J\x04\x08\x08\x10\t\"\xab\x01\n\x06Header\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x06 \x01(\t\x12\x0f\n\x07pattern\x18\r \x01(\tJ\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\n\x10\x0bJ\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\rJ\x04\x08\x0e\x10\x0fJ\x04\x08\x0f\x10\x10J\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13\"\xb8\x04\n\x08Response\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12?\n\x06schema\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema\x12O\n\x07headers\x18\x03 \x03(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.Response.HeadersEntry\x12Q\n\x08\x65xamples\x18\x04 \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Response.ExamplesEntry\x12U\n\nextensions\x18\x05 \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Response.ExtensionsEntry\x1a_\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Header:\x02\x38\x01\x1a/\n\rExamplesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"\xf9\x02\n\x04Info\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x18\n\x10terms_of_service\x18\x03 \x01(\t\x12\x41\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Contact\x12\x41\n\x07license\x18\x05 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.License\x12\x0f\n\x07version\x18\x06 \x01(\t\x12Q\n\nextensions\x18\x07 \x03(\x0b\x32=.grpc.gateway.protoc_gen_swagger.options.Info.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"3\n\x07\x43ontact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\r\n\x05\x65mail\x18\x03 \x01(\t\"$\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"9\n\x15\x45xternalDocumentation\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"\x9c\x02\n\x06Schema\x12H\n\x0bjson_schema\x18\x01 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchema\x12\x15\n\rdiscriminator\x18\x02 \x01(\t\x12\x11\n\tread_only\x18\x03 \x01(\x08\x12U\n\rexternal_docs\x18\x05 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12)\n\x07\x65xample\x18\x06 \x01(\x0b\x32\x14.google.protobuf.AnyB\x02\x18\x01\x12\x16\n\x0e\x65xample_string\x18\x07 \x01(\tJ\x04\x08\x04\x10\x05\"\xe3\x05\n\nJSONSchema\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\t\x12\x11\n\tread_only\x18\x08 \x01(\x08\x12\x0f\n\x07\x65xample\x18\t \x01(\t\x12\x13\n\x0bmultiple_of\x18\n \x01(\x01\x12\x0f\n\x07maximum\x18\x0b \x01(\x01\x12\x19\n\x11\x65xclusive_maximum\x18\x0c \x01(\x08\x12\x0f\n\x07minimum\x18\r \x01(\x01\x12\x19\n\x11\x65xclusive_minimum\x18\x0e \x01(\x08\x12\x12\n\nmax_length\x18\x0f \x01(\x04\x12\x12\n\nmin_length\x18\x10 \x01(\x04\x12\x0f\n\x07pattern\x18\x11 \x01(\t\x12\x11\n\tmax_items\x18\x14 \x01(\x04\x12\x11\n\tmin_items\x18\x15 \x01(\x04\x12\x14\n\x0cunique_items\x18\x16 \x01(\x08\x12\x16\n\x0emax_properties\x18\x18 \x01(\x04\x12\x16\n\x0emin_properties\x18\x19 \x01(\x04\x12\x10\n\x08required\x18\x1a \x03(\t\x12\r\n\x05\x61rray\x18\" \x03(\t\x12W\n\x04type\x18# \x03(\x0e\x32I.grpc.gateway.protoc_gen_swagger.options.JSONSchema.JSONSchemaSimpleTypes\x12\x0e\n\x06\x66ormat\x18$ \x01(\t\x12\x0c\n\x04\x65num\x18. \x03(\t\"w\n\x15JSONSchemaSimpleTypes\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41RRAY\x10\x01\x12\x0b\n\x07\x42OOLEAN\x10\x02\x12\x0b\n\x07INTEGER\x10\x03\x12\x08\n\x04NULL\x10\x04\x12\n\n\x06NUMBER\x10\x05\x12\n\n\x06OBJECT\x10\x06\x12\n\n\x06STRING\x10\x07J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13J\x04\x08\x13\x10\x14J\x04\x08\x17\x10\x18J\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1dJ\x04\x08\x1d\x10\x1eJ\x04\x08\x1e\x10\"J\x04\x08%\x10*J\x04\x08*\x10+J\x04\x08+\x10.\"w\n\x03Tag\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12U\n\rexternal_docs\x18\x03 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentationJ\x04\x08\x01\x10\x02\"\xdd\x01\n\x13SecurityDefinitions\x12\\\n\x08security\x18\x01 \x03(\x0b\x32J.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions.SecurityEntry\x1ah\n\rSecurityEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.grpc.gateway.protoc_gen_swagger.options.SecurityScheme:\x02\x38\x01\"\x96\x06\n\x0eSecurityScheme\x12J\n\x04type\x18\x01 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Type\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x46\n\x02in\x18\x04 \x01(\x0e\x32:.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.In\x12J\n\x04\x66low\x18\x05 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Flow\x12\x19\n\x11\x61uthorization_url\x18\x06 \x01(\t\x12\x11\n\ttoken_url\x18\x07 \x01(\t\x12?\n\x06scopes\x18\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Scopes\x12[\n\nextensions\x18\t \x03(\x0b\x32G.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"K\n\x04Type\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\x0e\n\nTYPE_BASIC\x10\x01\x12\x10\n\x0cTYPE_API_KEY\x10\x02\x12\x0f\n\x0bTYPE_OAUTH2\x10\x03\"1\n\x02In\x12\x0e\n\nIN_INVALID\x10\x00\x12\x0c\n\x08IN_QUERY\x10\x01\x12\r\n\tIN_HEADER\x10\x02\"j\n\x04\x46low\x12\x10\n\x0c\x46LOW_INVALID\x10\x00\x12\x11\n\rFLOW_IMPLICIT\x10\x01\x12\x11\n\rFLOW_PASSWORD\x10\x02\x12\x14\n\x10\x46LOW_APPLICATION\x10\x03\x12\x14\n\x10\x46LOW_ACCESS_CODE\x10\x04\"\xc9\x02\n\x13SecurityRequirement\x12s\n\x14security_requirement\x18\x01 \x03(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementEntry\x1a)\n\x18SecurityRequirementValue\x12\r\n\x05scope\x18\x01 \x03(\t\x1a\x91\x01\n\x18SecurityRequirementEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x64\n\x05value\x18\x02 \x01(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementValue:\x02\x38\x01\"\x81\x01\n\x06Scopes\x12I\n\x05scope\x18\x01 \x03(\x0b\x32:.grpc.gateway.protoc_gen_swagger.options.Scopes.ScopeEntry\x1a,\n\nScopeEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x43ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n*protoc-gen-swagger/options/openapiv2.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto"\xa0\x07\n\x07Swagger\x12\x0f\n\x07swagger\x18\x01 \x01(\t\x12;\n\x04info\x18\x02 \x01(\x0b\x32-.grpc.gateway.protoc_gen_swagger.options.Info\x12\x0c\n\x04host\x18\x03 \x01(\t\x12\x11\n\tbase_path\x18\x04 \x01(\t\x12O\n\x07schemes\x18\x05 \x03(\x0e\x32>.grpc.gateway.protoc_gen_swagger.options.Swagger.SwaggerScheme\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12R\n\tresponses\x18\n \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Swagger.ResponsesEntry\x12Z\n\x14security_definitions\x18\x0b \x01(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12U\n\rexternal_docs\x18\x0e \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12T\n\nextensions\x18\x0f \x03(\x0b\x32@.grpc.gateway.protoc_gen_swagger.options.Swagger.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"B\n\rSwaggerScheme\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04HTTP\x10\x01\x12\t\n\x05HTTPS\x10\x02\x12\x06\n\x02WS\x10\x03\x12\x07\n\x03WSS\x10\x04J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\r\x10\x0e"\xa9\x05\n\tOperation\x12\x0c\n\x04tags\x18\x01 \x03(\t\x12\x0f\n\x07summary\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12U\n\rexternal_docs\x18\x04 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12\x14\n\x0coperation_id\x18\x05 \x01(\t\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12T\n\tresponses\x18\t \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Operation.ResponsesEntry\x12\x0f\n\x07schemes\x18\n \x03(\t\x12\x12\n\ndeprecated\x18\x0b \x01(\x08\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12V\n\nextensions\x18\r \x03(\x0b\x32\x42.grpc.gateway.protoc_gen_swagger.options.Operation.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01J\x04\x08\x08\x10\t"\xab\x01\n\x06Header\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x06 \x01(\t\x12\x0f\n\x07pattern\x18\r \x01(\tJ\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\n\x10\x0bJ\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\rJ\x04\x08\x0e\x10\x0fJ\x04\x08\x0f\x10\x10J\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13"\xb8\x04\n\x08Response\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12?\n\x06schema\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema\x12O\n\x07headers\x18\x03 \x03(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.Response.HeadersEntry\x12Q\n\x08\x65xamples\x18\x04 \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Response.ExamplesEntry\x12U\n\nextensions\x18\x05 \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Response.ExtensionsEntry\x1a_\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Header:\x02\x38\x01\x1a/\n\rExamplesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"\xf9\x02\n\x04Info\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x18\n\x10terms_of_service\x18\x03 \x01(\t\x12\x41\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Contact\x12\x41\n\x07license\x18\x05 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.License\x12\x0f\n\x07version\x18\x06 \x01(\t\x12Q\n\nextensions\x18\x07 \x03(\x0b\x32=.grpc.gateway.protoc_gen_swagger.options.Info.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"3\n\x07\x43ontact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\r\n\x05\x65mail\x18\x03 \x01(\t"$\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t"9\n\x15\x45xternalDocumentation\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t"\x9c\x02\n\x06Schema\x12H\n\x0bjson_schema\x18\x01 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchema\x12\x15\n\rdiscriminator\x18\x02 \x01(\t\x12\x11\n\tread_only\x18\x03 \x01(\x08\x12U\n\rexternal_docs\x18\x05 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12)\n\x07\x65xample\x18\x06 \x01(\x0b\x32\x14.google.protobuf.AnyB\x02\x18\x01\x12\x16\n\x0e\x65xample_string\x18\x07 \x01(\tJ\x04\x08\x04\x10\x05"\xe3\x05\n\nJSONSchema\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\t\x12\x11\n\tread_only\x18\x08 \x01(\x08\x12\x0f\n\x07\x65xample\x18\t \x01(\t\x12\x13\n\x0bmultiple_of\x18\n \x01(\x01\x12\x0f\n\x07maximum\x18\x0b \x01(\x01\x12\x19\n\x11\x65xclusive_maximum\x18\x0c \x01(\x08\x12\x0f\n\x07minimum\x18\r \x01(\x01\x12\x19\n\x11\x65xclusive_minimum\x18\x0e \x01(\x08\x12\x12\n\nmax_length\x18\x0f \x01(\x04\x12\x12\n\nmin_length\x18\x10 \x01(\x04\x12\x0f\n\x07pattern\x18\x11 \x01(\t\x12\x11\n\tmax_items\x18\x14 \x01(\x04\x12\x11\n\tmin_items\x18\x15 \x01(\x04\x12\x14\n\x0cunique_items\x18\x16 \x01(\x08\x12\x16\n\x0emax_properties\x18\x18 \x01(\x04\x12\x16\n\x0emin_properties\x18\x19 \x01(\x04\x12\x10\n\x08required\x18\x1a \x03(\t\x12\r\n\x05\x61rray\x18" \x03(\t\x12W\n\x04type\x18# \x03(\x0e\x32I.grpc.gateway.protoc_gen_swagger.options.JSONSchema.JSONSchemaSimpleTypes\x12\x0e\n\x06\x66ormat\x18$ \x01(\t\x12\x0c\n\x04\x65num\x18. \x03(\t"w\n\x15JSONSchemaSimpleTypes\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41RRAY\x10\x01\x12\x0b\n\x07\x42OOLEAN\x10\x02\x12\x0b\n\x07INTEGER\x10\x03\x12\x08\n\x04NULL\x10\x04\x12\n\n\x06NUMBER\x10\x05\x12\n\n\x06OBJECT\x10\x06\x12\n\n\x06STRING\x10\x07J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13J\x04\x08\x13\x10\x14J\x04\x08\x17\x10\x18J\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1dJ\x04\x08\x1d\x10\x1eJ\x04\x08\x1e\x10"J\x04\x08%\x10*J\x04\x08*\x10+J\x04\x08+\x10."w\n\x03Tag\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12U\n\rexternal_docs\x18\x03 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentationJ\x04\x08\x01\x10\x02"\xdd\x01\n\x13SecurityDefinitions\x12\\\n\x08security\x18\x01 \x03(\x0b\x32J.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions.SecurityEntry\x1ah\n\rSecurityEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.grpc.gateway.protoc_gen_swagger.options.SecurityScheme:\x02\x38\x01"\x96\x06\n\x0eSecurityScheme\x12J\n\x04type\x18\x01 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Type\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x46\n\x02in\x18\x04 \x01(\x0e\x32:.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.In\x12J\n\x04\x66low\x18\x05 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Flow\x12\x19\n\x11\x61uthorization_url\x18\x06 \x01(\t\x12\x11\n\ttoken_url\x18\x07 \x01(\t\x12?\n\x06scopes\x18\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Scopes\x12[\n\nextensions\x18\t \x03(\x0b\x32G.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"K\n\x04Type\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\x0e\n\nTYPE_BASIC\x10\x01\x12\x10\n\x0cTYPE_API_KEY\x10\x02\x12\x0f\n\x0bTYPE_OAUTH2\x10\x03"1\n\x02In\x12\x0e\n\nIN_INVALID\x10\x00\x12\x0c\n\x08IN_QUERY\x10\x01\x12\r\n\tIN_HEADER\x10\x02"j\n\x04\x46low\x12\x10\n\x0c\x46LOW_INVALID\x10\x00\x12\x11\n\rFLOW_IMPLICIT\x10\x01\x12\x11\n\rFLOW_PASSWORD\x10\x02\x12\x14\n\x10\x46LOW_APPLICATION\x10\x03\x12\x14\n\x10\x46LOW_ACCESS_CODE\x10\x04"\xc9\x02\n\x13SecurityRequirement\x12s\n\x14security_requirement\x18\x01 \x03(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementEntry\x1a)\n\x18SecurityRequirementValue\x12\r\n\x05scope\x18\x01 \x03(\t\x1a\x91\x01\n\x18SecurityRequirementEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x64\n\x05value\x18\x02 \x01(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementValue:\x02\x38\x01"\x81\x01\n\x06Scopes\x12I\n\x05scope\x18\x01 \x03(\x0b\x32:.grpc.gateway.protoc_gen_swagger.options.Scopes.ScopeEntry\x1a,\n\nScopeEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x43ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3' +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.openapiv2_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' - _SWAGGER_RESPONSESENTRY._options = None - _SWAGGER_RESPONSESENTRY._serialized_options = b'8\001' - _SWAGGER_EXTENSIONSENTRY._options = None - _SWAGGER_EXTENSIONSENTRY._serialized_options = b'8\001' - _OPERATION_RESPONSESENTRY._options = None - _OPERATION_RESPONSESENTRY._serialized_options = b'8\001' - _OPERATION_EXTENSIONSENTRY._options = None - _OPERATION_EXTENSIONSENTRY._serialized_options = b'8\001' - _RESPONSE_HEADERSENTRY._options = None - _RESPONSE_HEADERSENTRY._serialized_options = b'8\001' - _RESPONSE_EXAMPLESENTRY._options = None - _RESPONSE_EXAMPLESENTRY._serialized_options = b'8\001' - _RESPONSE_EXTENSIONSENTRY._options = None - _RESPONSE_EXTENSIONSENTRY._serialized_options = b'8\001' - _INFO_EXTENSIONSENTRY._options = None - _INFO_EXTENSIONSENTRY._serialized_options = b'8\001' - _SCHEMA.fields_by_name['example']._options = None - _SCHEMA.fields_by_name['example']._serialized_options = b'\030\001' - _SECURITYDEFINITIONS_SECURITYENTRY._options = None - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_options = b'8\001' - _SECURITYSCHEME_EXTENSIONSENTRY._options = None - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_options = b'8\001' - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._options = None - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_options = b'8\001' - _SCOPES_SCOPEENTRY._options = None - _SCOPES_SCOPEENTRY._serialized_options = b'8\001' - _SWAGGER._serialized_start=145 - _SWAGGER._serialized_end=1073 - _SWAGGER_RESPONSESENTRY._serialized_start=813 - _SWAGGER_RESPONSESENTRY._serialized_end=912 - _SWAGGER_EXTENSIONSENTRY._serialized_start=914 - _SWAGGER_EXTENSIONSENTRY._serialized_end=987 - _SWAGGER_SWAGGERSCHEME._serialized_start=989 - _SWAGGER_SWAGGERSCHEME._serialized_end=1055 - _OPERATION._serialized_start=1076 - _OPERATION._serialized_end=1757 - _OPERATION_RESPONSESENTRY._serialized_start=813 - _OPERATION_RESPONSESENTRY._serialized_end=912 - _OPERATION_EXTENSIONSENTRY._serialized_start=914 - _OPERATION_EXTENSIONSENTRY._serialized_end=987 - _HEADER._serialized_start=1760 - _HEADER._serialized_end=1931 - _RESPONSE._serialized_start=1934 - _RESPONSE._serialized_end=2502 - _RESPONSE_HEADERSENTRY._serialized_start=2283 - _RESPONSE_HEADERSENTRY._serialized_end=2378 - _RESPONSE_EXAMPLESENTRY._serialized_start=2380 - _RESPONSE_EXAMPLESENTRY._serialized_end=2427 - _RESPONSE_EXTENSIONSENTRY._serialized_start=914 - _RESPONSE_EXTENSIONSENTRY._serialized_end=987 - _INFO._serialized_start=2505 - _INFO._serialized_end=2882 - _INFO_EXTENSIONSENTRY._serialized_start=914 - _INFO_EXTENSIONSENTRY._serialized_end=987 - _CONTACT._serialized_start=2884 - _CONTACT._serialized_end=2935 - _LICENSE._serialized_start=2937 - _LICENSE._serialized_end=2973 - _EXTERNALDOCUMENTATION._serialized_start=2975 - _EXTERNALDOCUMENTATION._serialized_end=3032 - _SCHEMA._serialized_start=3035 - _SCHEMA._serialized_end=3319 - _JSONSCHEMA._serialized_start=3322 - _JSONSCHEMA._serialized_end=4061 - _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_start=3864 - _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_end=3983 - _TAG._serialized_start=4063 - _TAG._serialized_end=4182 - _SECURITYDEFINITIONS._serialized_start=4185 - _SECURITYDEFINITIONS._serialized_end=4406 - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_start=4302 - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_end=4406 - _SECURITYSCHEME._serialized_start=4409 - _SECURITYSCHEME._serialized_end=5199 - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_start=914 - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_end=987 - _SECURITYSCHEME_TYPE._serialized_start=4965 - _SECURITYSCHEME_TYPE._serialized_end=5040 - _SECURITYSCHEME_IN._serialized_start=5042 - _SECURITYSCHEME_IN._serialized_end=5091 - _SECURITYSCHEME_FLOW._serialized_start=5093 - _SECURITYSCHEME_FLOW._serialized_end=5199 - _SECURITYREQUIREMENT._serialized_start=5202 - _SECURITYREQUIREMENT._serialized_end=5531 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_start=5342 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_end=5383 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_start=5386 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_end=5531 - _SCOPES._serialized_start=5534 - _SCOPES._serialized_end=5663 - _SCOPES_SCOPEENTRY._serialized_start=5619 - _SCOPES_SCOPEENTRY._serialized_end=5663 + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' + _SWAGGER_RESPONSESENTRY._options = None + _SWAGGER_RESPONSESENTRY._serialized_options = b'8\001' + _SWAGGER_EXTENSIONSENTRY._options = None + _SWAGGER_EXTENSIONSENTRY._serialized_options = b'8\001' + _OPERATION_RESPONSESENTRY._options = None + _OPERATION_RESPONSESENTRY._serialized_options = b'8\001' + _OPERATION_EXTENSIONSENTRY._options = None + _OPERATION_EXTENSIONSENTRY._serialized_options = b'8\001' + _RESPONSE_HEADERSENTRY._options = None + _RESPONSE_HEADERSENTRY._serialized_options = b'8\001' + _RESPONSE_EXAMPLESENTRY._options = None + _RESPONSE_EXAMPLESENTRY._serialized_options = b'8\001' + _RESPONSE_EXTENSIONSENTRY._options = None + _RESPONSE_EXTENSIONSENTRY._serialized_options = b'8\001' + _INFO_EXTENSIONSENTRY._options = None + _INFO_EXTENSIONSENTRY._serialized_options = b'8\001' + _SCHEMA.fields_by_name['example']._options = None + _SCHEMA.fields_by_name['example']._serialized_options = b'\030\001' + _SECURITYDEFINITIONS_SECURITYENTRY._options = None + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_options = b'8\001' + _SECURITYSCHEME_EXTENSIONSENTRY._options = None + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_options = b'8\001' + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._options = None + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_options = b'8\001' + _SCOPES_SCOPEENTRY._options = None + _SCOPES_SCOPEENTRY._serialized_options = b'8\001' + _SWAGGER._serialized_start = 145 + _SWAGGER._serialized_end = 1073 + _SWAGGER_RESPONSESENTRY._serialized_start = 813 + _SWAGGER_RESPONSESENTRY._serialized_end = 912 + _SWAGGER_EXTENSIONSENTRY._serialized_start = 914 + _SWAGGER_EXTENSIONSENTRY._serialized_end = 987 + _SWAGGER_SWAGGERSCHEME._serialized_start = 989 + _SWAGGER_SWAGGERSCHEME._serialized_end = 1055 + _OPERATION._serialized_start = 1076 + _OPERATION._serialized_end = 1757 + _OPERATION_RESPONSESENTRY._serialized_start = 813 + _OPERATION_RESPONSESENTRY._serialized_end = 912 + _OPERATION_EXTENSIONSENTRY._serialized_start = 914 + _OPERATION_EXTENSIONSENTRY._serialized_end = 987 + _HEADER._serialized_start = 1760 + _HEADER._serialized_end = 1931 + _RESPONSE._serialized_start = 1934 + _RESPONSE._serialized_end = 2502 + _RESPONSE_HEADERSENTRY._serialized_start = 2283 + _RESPONSE_HEADERSENTRY._serialized_end = 2378 + _RESPONSE_EXAMPLESENTRY._serialized_start = 2380 + _RESPONSE_EXAMPLESENTRY._serialized_end = 2427 + _RESPONSE_EXTENSIONSENTRY._serialized_start = 914 + _RESPONSE_EXTENSIONSENTRY._serialized_end = 987 + _INFO._serialized_start = 2505 + _INFO._serialized_end = 2882 + _INFO_EXTENSIONSENTRY._serialized_start = 914 + _INFO_EXTENSIONSENTRY._serialized_end = 987 + _CONTACT._serialized_start = 2884 + _CONTACT._serialized_end = 2935 + _LICENSE._serialized_start = 2937 + _LICENSE._serialized_end = 2973 + _EXTERNALDOCUMENTATION._serialized_start = 2975 + _EXTERNALDOCUMENTATION._serialized_end = 3032 + _SCHEMA._serialized_start = 3035 + _SCHEMA._serialized_end = 3319 + _JSONSCHEMA._serialized_start = 3322 + _JSONSCHEMA._serialized_end = 4061 + _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_start = 3864 + _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_end = 3983 + _TAG._serialized_start = 4063 + _TAG._serialized_end = 4182 + _SECURITYDEFINITIONS._serialized_start = 4185 + _SECURITYDEFINITIONS._serialized_end = 4406 + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_start = 4302 + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_end = 4406 + _SECURITYSCHEME._serialized_start = 4409 + _SECURITYSCHEME._serialized_end = 5199 + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_start = 914 + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_end = 987 + _SECURITYSCHEME_TYPE._serialized_start = 4965 + _SECURITYSCHEME_TYPE._serialized_end = 5040 + _SECURITYSCHEME_IN._serialized_start = 5042 + _SECURITYSCHEME_IN._serialized_end = 5091 + _SECURITYSCHEME_FLOW._serialized_start = 5093 + _SECURITYSCHEME_FLOW._serialized_end = 5199 + _SECURITYREQUIREMENT._serialized_start = 5202 + _SECURITYREQUIREMENT._serialized_end = 5531 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_start = 5342 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_end = 5383 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_start = 5386 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_end = 5531 + _SCOPES._serialized_start = 5534 + _SCOPES._serialized_end = 5663 + _SCOPES_SCOPEENTRY._serialized_start = 5619 + _SCOPES_SCOPEENTRY._serialized_end = 5663 # @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py index 2daafffe..bf947056 100644 --- a/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py +++ b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py @@ -1,4 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" -import grpc +import grpc diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index f691244c..3a0140c9 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -1,25 +1,25 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ -__version__ = "1.20.0" +__version__ = '1.19.6' diff --git a/src/scanoss/api/__init__.py b/src/scanoss/api/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/__init__.py +++ b/src/scanoss/api/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/common/__init__.py b/src/scanoss/api/common/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/common/__init__.py +++ b/src/scanoss/api/common/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/common/v2/__init__.py b/src/scanoss/api/common/v2/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/common/v2/__init__.py +++ b/src/scanoss/api/common/v2/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index 23546c71..9cb92316 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/common/v2/scanoss-common.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -11,26 +12,25 @@ _sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3' +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _STATUSCODE._serialized_start=336 - _STATUSCODE._serialized_end=432 - _STATUSRESPONSE._serialized_start=69 - _STATUSRESPONSE._serialized_end=153 - _ECHOREQUEST._serialized_start=155 - _ECHOREQUEST._serialized_end=185 - _ECHORESPONSE._serialized_start=187 - _ECHORESPONSE._serialized_end=218 - _PURLREQUEST._serialized_start=220 - _PURLREQUEST._serialized_end=334 - _PURLREQUEST_PURLS._serialized_start=292 - _PURLREQUEST_PURLS._serialized_end=334 + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' + _STATUSCODE._serialized_start = 336 + _STATUSCODE._serialized_end = 432 + _STATUSRESPONSE._serialized_start = 69 + _STATUSRESPONSE._serialized_end = 153 + _ECHOREQUEST._serialized_start = 155 + _ECHOREQUEST._serialized_end = 185 + _ECHORESPONSE._serialized_start = 187 + _ECHORESPONSE._serialized_end = 218 + _PURLREQUEST._serialized_start = 220 + _PURLREQUEST._serialized_end = 334 + _PURLREQUEST_PURLS._serialized_start = 292 + _PURLREQUEST_PURLS._serialized_end = 334 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index 2daafffe..bf947056 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -1,4 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" -import grpc +import grpc diff --git a/src/scanoss/api/components/__init__.py b/src/scanoss/api/components/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/components/__init__.py +++ b/src/scanoss/api/components/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/components/v2/__init__.py b/src/scanoss/api/components/v2/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/components/v2/__init__.py +++ b/src/scanoss/api/components/v2/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index cf1290a9..04938b6d 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/components/v2/scanoss-components.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,46 +17,55 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05\"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xd4\x04\n\nComponents\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/components/echo:\x01*\x12\x95\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/components/search:\x01*\x12\x9d\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"&\x82\xd3\xe4\x93\x02 \"\x1b/api/v2/components/versions:\x01*\x12\x98\x01\n\x16GetComponentStatistics\x12\".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse\"(\x82\xd3\xe4\x93\x02\"\"\x1d/api/v2/components/statistics:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Components Service\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xd4\x04\n\nComponents\x12s\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse""\x82\xd3\xe4\x93\x02\x1c"\x17/api/v2/components/echo:\x01*\x12\x95\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse"$\x82\xd3\xe4\x93\x02\x1e"\x19/api/v2/components/search:\x01*\x12\x9d\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse"&\x82\xd3\xe4\x93\x02 "\x1b/api/v2/components/versions:\x01*\x12\x98\x01\n\x16GetComponentStatistics\x12".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse"(\x82\xd3\xe4\x93\x02""\x1d/api/v2/components/statistics:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Components Service"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\331\001\022s\n\032SCANOSS Components Service\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _COMPONENTS.methods_by_name['Echo']._options = None - _COMPONENTS.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/components/echo:\001*' - _COMPONENTS.methods_by_name['SearchComponents']._options = None - _COMPONENTS.methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/components/search:\001*' - _COMPONENTS.methods_by_name['GetComponentVersions']._options = None - _COMPONENTS.methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002 \"\033/api/v2/components/versions:\001*' - _COMPONENTS.methods_by_name['GetComponentStatistics']._options = None - _COMPONENTS.methods_by_name['GetComponentStatistics']._serialized_options = b'\202\323\344\223\002\"\"\035/api/v2/components/statistics:\001*' - _COMPSEARCHREQUEST._serialized_start=201 - _COMPSEARCHREQUEST._serialized_end=319 - _COMPSTATISTIC._serialized_start=322 - _COMPSTATISTIC._serialized_end=524 - _COMPSTATISTIC_LANGUAGE._serialized_start=485 - _COMPSTATISTIC_LANGUAGE._serialized_end=524 - _COMPSTATISTICRESPONSE._serialized_start=527 - _COMPSTATISTICRESPONSE._serialized_end=778 - _COMPSTATISTICRESPONSE_PURLS._serialized_start=678 - _COMPSTATISTICRESPONSE_PURLS._serialized_end=778 - _COMPSEARCHRESPONSE._serialized_start=781 - _COMPSEARCHRESPONSE._serialized_end=992 - _COMPSEARCHRESPONSE_COMPONENT._serialized_start=935 - _COMPSEARCHRESPONSE_COMPONENT._serialized_end=992 - _COMPVERSIONREQUEST._serialized_start=994 - _COMPVERSIONREQUEST._serialized_end=1043 - _COMPVERSIONRESPONSE._serialized_start=1046 - _COMPVERSIONRESPONSE._serialized_end=1516 - _COMPVERSIONRESPONSE_LICENSE._serialized_start=1201 - _COMPVERSIONRESPONSE_LICENSE._serialized_end=1280 - _COMPVERSIONRESPONSE_VERSION._serialized_start=1282 - _COMPVERSIONRESPONSE_VERSION._serialized_end=1382 - _COMPVERSIONRESPONSE_COMPONENT._serialized_start=1385 - _COMPVERSIONRESPONSE_COMPONENT._serialized_end=1516 - _COMPONENTS._serialized_start=1519 - _COMPONENTS._serialized_end=2115 + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\331\001\022s\n\032SCANOSS Components Service"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _COMPONENTS.methods_by_name['Echo']._options = None + _COMPONENTS.methods_by_name[ + 'Echo' + ]._serialized_options = b'\202\323\344\223\002\034"\027/api/v2/components/echo:\001*' + _COMPONENTS.methods_by_name['SearchComponents']._options = None + _COMPONENTS.methods_by_name[ + 'SearchComponents' + ]._serialized_options = b'\202\323\344\223\002\036"\031/api/v2/components/search:\001*' + _COMPONENTS.methods_by_name['GetComponentVersions']._options = None + _COMPONENTS.methods_by_name[ + 'GetComponentVersions' + ]._serialized_options = b'\202\323\344\223\002 "\033/api/v2/components/versions:\001*' + _COMPONENTS.methods_by_name['GetComponentStatistics']._options = None + _COMPONENTS.methods_by_name[ + 'GetComponentStatistics' + ]._serialized_options = b'\202\323\344\223\002""\035/api/v2/components/statistics:\001*' + _COMPSEARCHREQUEST._serialized_start = 201 + _COMPSEARCHREQUEST._serialized_end = 319 + _COMPSTATISTIC._serialized_start = 322 + _COMPSTATISTIC._serialized_end = 524 + _COMPSTATISTIC_LANGUAGE._serialized_start = 485 + _COMPSTATISTIC_LANGUAGE._serialized_end = 524 + _COMPSTATISTICRESPONSE._serialized_start = 527 + _COMPSTATISTICRESPONSE._serialized_end = 778 + _COMPSTATISTICRESPONSE_PURLS._serialized_start = 678 + _COMPSTATISTICRESPONSE_PURLS._serialized_end = 778 + _COMPSEARCHRESPONSE._serialized_start = 781 + _COMPSEARCHRESPONSE._serialized_end = 992 + _COMPSEARCHRESPONSE_COMPONENT._serialized_start = 935 + _COMPSEARCHRESPONSE_COMPONENT._serialized_end = 992 + _COMPVERSIONREQUEST._serialized_start = 994 + _COMPVERSIONREQUEST._serialized_end = 1043 + _COMPVERSIONRESPONSE._serialized_start = 1046 + _COMPVERSIONRESPONSE._serialized_end = 1516 + _COMPVERSIONRESPONSE_LICENSE._serialized_start = 1201 + _COMPVERSIONRESPONSE_LICENSE._serialized_end = 1280 + _COMPVERSIONRESPONSE_VERSION._serialized_start = 1282 + _COMPVERSIONRESPONSE_VERSION._serialized_end = 1382 + _COMPVERSIONRESPONSE_COMPONENT._serialized_start = 1385 + _COMPVERSIONRESPONSE_COMPONENT._serialized_end = 1516 + _COMPONENTS._serialized_start = 1519 + _COMPONENTS._serialized_end = 2115 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py index 8ac7b92c..da8d455a 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py @@ -1,9 +1,12 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" + import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from scanoss.api.components.v2 import scanoss_components_pb2 as scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2 +from scanoss.api.components.v2 import ( + scanoss_components_pb2 as scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2, +) class ComponentsStub(object): @@ -18,25 +21,25 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.components.v2.Components/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.components.v2.Components/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.SearchComponents = channel.unary_unary( - '/scanoss.api.components.v2.Components/SearchComponents', - request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, - ) + '/scanoss.api.components.v2.Components/SearchComponents', + request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, + ) self.GetComponentVersions = channel.unary_unary( - '/scanoss.api.components.v2.Components/GetComponentVersions', - request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, - ) + '/scanoss.api.components.v2.Components/GetComponentVersions', + request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, + ) self.GetComponentStatistics = channel.unary_unary( - '/scanoss.api.components.v2.Components/GetComponentStatistics', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, - ) + '/scanoss.api.components.v2.Components/GetComponentStatistics', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, + ) class ComponentsServicer(object): @@ -45,29 +48,25 @@ class ComponentsServicer(object): """ def Echo(self, request, context): - """Standard echo - """ + """Standard echo""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def SearchComponents(self, request, context): - """Search for components - """ + """Search for components""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetComponentVersions(self, request, context): - """Get all version information for a specific component - """ + """Get all version information for a specific component""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetComponentStatistics(self, request, context): - """Get the statistics for the specified components - """ + """Get the statistics for the specified components""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -75,102 +74,149 @@ def GetComponentStatistics(self, request, context): def add_ComponentsServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'SearchComponents': grpc.unary_unary_rpc_method_handler( - servicer.SearchComponents, - request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.FromString, - response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.SerializeToString, - ), - 'GetComponentVersions': grpc.unary_unary_rpc_method_handler( - servicer.GetComponentVersions, - request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.FromString, - response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.SerializeToString, - ), - 'GetComponentStatistics': grpc.unary_unary_rpc_method_handler( - servicer.GetComponentStatistics, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'SearchComponents': grpc.unary_unary_rpc_method_handler( + servicer.SearchComponents, + request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.SerializeToString, + ), + 'GetComponentVersions': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentVersions, + request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.SerializeToString, + ), + 'GetComponentStatistics': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentStatistics, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.SerializeToString, + ), } - generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.components.v2.Components', rpc_method_handlers) + generic_handler = grpc.method_handlers_generic_handler('scanoss.api.components.v2.Components', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) - # This class is part of an EXPERIMENTAL API. +# This class is part of an EXPERIMENTAL API. class Components(object): """ Expose all of the SCANOSS Component RPCs here """ @staticmethod - def Echo(request, + def Echo( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/Echo', + '/scanoss.api.components.v2.Components/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) @staticmethod - def SearchComponents(request, + def SearchComponents( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/SearchComponents', + '/scanoss.api.components.v2.Components/SearchComponents', scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) @staticmethod - def GetComponentVersions(request, + def GetComponentVersions( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/GetComponentVersions', + '/scanoss.api.components.v2.Components/GetComponentVersions', scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) @staticmethod - def GetComponentStatistics(request, + def GetComponentStatistics( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/GetComponentStatistics', + '/scanoss.api.components.v2.Components/GetComponentStatistics', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py index 3431e10e..577b9af2 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/cryptography/v2/scanoss-cryptography.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,24 +17,29 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xb9\x02\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x31\n\nAlgorithms\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\x1au\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12M\n\nalgorithms\x18\x03 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.AlgorithmResponse.Algorithms2\x97\x02\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/cryptography/algorithms:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"\xb9\x02\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x31\n\nAlgorithms\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\x1au\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12M\n\nalgorithms\x18\x03 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.AlgorithmResponse.Algorithms2\x97\x02\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse"$\x82\xd3\xe4\x93\x02\x1e"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse"*\x82\xd3\xe4\x93\x02$"\x1f/api/v2/cryptography/algorithms:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _CRYPTOGRAPHY.methods_by_name['Echo']._options = None - _CRYPTOGRAPHY.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/cryptography/echo:\001*' - _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._options = None - _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/cryptography/algorithms:\001*' - _ALGORITHMRESPONSE._serialized_start=208 - _ALGORITHMRESPONSE._serialized_end=521 - _ALGORITHMRESPONSE_ALGORITHMS._serialized_start=353 - _ALGORITHMRESPONSE_ALGORITHMS._serialized_end=402 - _ALGORITHMRESPONSE_PURLS._serialized_start=404 - _ALGORITHMRESPONSE_PURLS._serialized_end=521 - _CRYPTOGRAPHY._serialized_start=524 - _CRYPTOGRAPHY._serialized_end=803 + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _CRYPTOGRAPHY.methods_by_name['Echo']._options = None + _CRYPTOGRAPHY.methods_by_name[ + 'Echo' + ]._serialized_options = b'\202\323\344\223\002\036"\031/api/v2/cryptography/echo:\001*' + _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._options = None + _CRYPTOGRAPHY.methods_by_name[ + 'GetAlgorithms' + ]._serialized_options = b'\202\323\344\223\002$"\037/api/v2/cryptography/algorithms:\001*' + _ALGORITHMRESPONSE._serialized_start = 208 + _ALGORITHMRESPONSE._serialized_end = 521 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_start = 353 + _ALGORITHMRESPONSE_ALGORITHMS._serialized_end = 402 + _ALGORITHMRESPONSE_PURLS._serialized_start = 404 + _ALGORITHMRESPONSE_PURLS._serialized_end = 521 + _CRYPTOGRAPHY._serialized_start = 524 + _CRYPTOGRAPHY._serialized_end = 803 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py index 1d641e82..22a866a8 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py @@ -1,9 +1,12 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" + import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from scanoss.api.cryptography.v2 import scanoss_cryptography_pb2 as scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2 +from scanoss.api.cryptography.v2 import ( + scanoss_cryptography_pb2 as scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2, +) class CryptographyStub(object): @@ -18,15 +21,15 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.cryptography.v2.Cryptography/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.cryptography.v2.Cryptography/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.GetAlgorithms = channel.unary_unary( - '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, - ) + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, + ) class CryptographyServicer(object): @@ -35,15 +38,13 @@ class CryptographyServicer(object): """ def Echo(self, request, context): - """Standard echo - """ + """Standard echo""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetAlgorithms(self, request, context): - """Get Cryptographic algorithms associated with a list of PURLs - """ + """Get Cryptographic algorithms associated with a list of PURLs""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -51,58 +52,83 @@ def GetAlgorithms(self, request, context): def add_CryptographyServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'GetAlgorithms': grpc.unary_unary_rpc_method_handler( - servicer.GetAlgorithms, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetAlgorithms': grpc.unary_unary_rpc_method_handler( + servicer.GetAlgorithms, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers) + 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers + ) server.add_generic_rpc_handlers((generic_handler,)) - # This class is part of an EXPERIMENTAL API. +# This class is part of an EXPERIMENTAL API. class Cryptography(object): """ Expose all of the SCANOSS Cryptography RPCs here """ @staticmethod - def Echo(request, + def Echo( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/Echo', + '/scanoss.api.cryptography.v2.Cryptography/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) @staticmethod - def GetAlgorithms(request, + def GetAlgorithms( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/src/scanoss/api/dependencies/__init__.py b/src/scanoss/api/dependencies/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/dependencies/__init__.py +++ b/src/scanoss/api/dependencies/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/dependencies/v2/__init__.py b/src/scanoss/api/dependencies/v2/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/dependencies/v2/__init__.py +++ b/src/scanoss/api/dependencies/v2/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index 0090b4cd..c1915e39 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/dependencies/v2/scanoss-dependencies.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,32 +17,37 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xa8\x02\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xa8\x02\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse"$\x82\xd3\xe4\x93\x02\x1e"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse",\x82\xd3\xe4\x93\x02&"!/api/v2/dependencies/dependencies:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service\"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _DEPENDENCIES.methods_by_name['Echo']._options = None - _DEPENDENCIES.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' - _DEPENDENCIES.methods_by_name['GetDependencies']._options = None - _DEPENDENCIES.methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' - _DEPENDENCYREQUEST._serialized_start=208 - _DEPENDENCYREQUEST._serialized_end=447 - _DEPENDENCYREQUEST_PURLS._serialized_start=313 - _DEPENDENCYREQUEST_PURLS._serialized_end=355 - _DEPENDENCYREQUEST_FILES._serialized_start=357 - _DEPENDENCYREQUEST_FILES._serialized_end=447 - _DEPENDENCYRESPONSE._serialized_start=450 - _DEPENDENCYRESPONSE._serialized_end=986 - _DEPENDENCYRESPONSE_LICENSES._serialized_start=597 - _DEPENDENCYRESPONSE_LICENSES._serialized_end=677 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=680 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=850 - _DEPENDENCYRESPONSE_FILES._serialized_start=853 - _DEPENDENCYRESPONSE_FILES._serialized_end=986 - _DEPENDENCIES._serialized_start=989 - _DEPENDENCIES._serialized_end=1285 + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _DEPENDENCIES.methods_by_name['Echo']._options = None + _DEPENDENCIES.methods_by_name[ + 'Echo' + ]._serialized_options = b'\202\323\344\223\002\036"\031/api/v2/dependencies/echo:\001*' + _DEPENDENCIES.methods_by_name['GetDependencies']._options = None + _DEPENDENCIES.methods_by_name[ + 'GetDependencies' + ]._serialized_options = b'\202\323\344\223\002&"!/api/v2/dependencies/dependencies:\001*' + _DEPENDENCYREQUEST._serialized_start = 208 + _DEPENDENCYREQUEST._serialized_end = 447 + _DEPENDENCYREQUEST_PURLS._serialized_start = 313 + _DEPENDENCYREQUEST_PURLS._serialized_end = 355 + _DEPENDENCYREQUEST_FILES._serialized_start = 357 + _DEPENDENCYREQUEST_FILES._serialized_end = 447 + _DEPENDENCYRESPONSE._serialized_start = 450 + _DEPENDENCYRESPONSE._serialized_end = 986 + _DEPENDENCYRESPONSE_LICENSES._serialized_start = 597 + _DEPENDENCYRESPONSE_LICENSES._serialized_end = 677 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start = 680 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end = 850 + _DEPENDENCYRESPONSE_FILES._serialized_start = 853 + _DEPENDENCYRESPONSE_FILES._serialized_end = 986 + _DEPENDENCIES._serialized_start = 989 + _DEPENDENCIES._serialized_end = 1285 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py index cddf7cfa..53d610f7 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py @@ -1,9 +1,12 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" + import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from scanoss.api.dependencies.v2 import scanoss_dependencies_pb2 as scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2 +from scanoss.api.dependencies.v2 import ( + scanoss_dependencies_pb2 as scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2, +) class DependenciesStub(object): @@ -18,15 +21,15 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.dependencies.v2.Dependencies/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.dependencies.v2.Dependencies/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.GetDependencies = channel.unary_unary( - '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', - request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, - ) + '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', + request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, + ) class DependenciesServicer(object): @@ -35,15 +38,13 @@ class DependenciesServicer(object): """ def Echo(self, request, context): - """Standard echo - """ + """Standard echo""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetDependencies(self, request, context): - """Get dependency details - """ + """Get dependency details""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -51,58 +52,83 @@ def GetDependencies(self, request, context): def add_DependenciesServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'GetDependencies': grpc.unary_unary_rpc_method_handler( - servicer.GetDependencies, - request_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.FromString, - response_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetDependencies': grpc.unary_unary_rpc_method_handler( + servicer.GetDependencies, + request_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.FromString, + response_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.dependencies.v2.Dependencies', rpc_method_handlers) + 'scanoss.api.dependencies.v2.Dependencies', rpc_method_handlers + ) server.add_generic_rpc_handlers((generic_handler,)) - # This class is part of an EXPERIMENTAL API. +# This class is part of an EXPERIMENTAL API. class Dependencies(object): """ Expose all of the SCANOSS Dependency RPCs here """ @staticmethod - def Echo(request, + def Echo( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/Echo', + '/scanoss.api.dependencies.v2.Dependencies/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) @staticmethod - def GetDependencies(request, + def GetDependencies( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', + '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/src/scanoss/api/scanning/__init__.py b/src/scanoss/api/scanning/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/scanning/__init__.py +++ b/src/scanoss/api/scanning/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/scanning/v2/__init__.py b/src/scanoss/api/scanning/v2/__init__.py index d2ed05b0..c67e64da 100644 --- a/src/scanoss/api/scanning/v2/__init__.py +++ b/src/scanoss/api/scanning/v2/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 60c795dd..48331ce3 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/scanning/v2/scanoss-scanning.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,16 +17,17 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto2}\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto2}\n\x08Scanning\x12q\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse" \x82\xd3\xe4\x93\x02\x1a"\x15/api/v2/scanning/echo:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service\"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _SCANNING.methods_by_name['Echo']._options = None - _SCANNING.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' - _SCANNING._serialized_start=195 - _SCANNING._serialized_end=320 + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _SCANNING.methods_by_name['Echo']._options = None + _SCANNING.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032"\025/api/v2/scanning/echo:\001*' + _SCANNING._serialized_start = 195 + _SCANNING._serialized_end = 320 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index f6530e94..9bf113b2 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -1,13 +1,13 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" + import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 class ScanningStub(object): - """Expose all of the SCANOSS Scanning RPCs here - """ + """Expose all of the SCANOSS Scanning RPCs here""" def __init__(self, channel): """Constructor. @@ -16,19 +16,17 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.scanning.v2.Scanning/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.scanning.v2.Scanning/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) class ScanningServicer(object): - """Expose all of the SCANOSS Scanning RPCs here - """ + """Expose all of the SCANOSS Scanning RPCs here""" def Echo(self, request, context): - """Standard echo - """ + """Standard echo""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -36,35 +34,45 @@ def Echo(self, request, context): def add_ScanningServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), } - generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.scanning.v2.Scanning', rpc_method_handlers) + generic_handler = grpc.method_handlers_generic_handler('scanoss.api.scanning.v2.Scanning', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) - # This class is part of an EXPERIMENTAL API. +# This class is part of an EXPERIMENTAL API. class Scanning(object): - """Expose all of the SCANOSS Scanning RPCs here - """ + """Expose all of the SCANOSS Scanning RPCs here""" @staticmethod - def Echo(request, + def Echo( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/Echo', + '/scanoss.api.scanning.v2.Scanning/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/src/scanoss/api/semgrep/__init__.py b/src/scanoss/api/semgrep/__init__.py index 31c0cbaa..da9b8e79 100644 --- a/src/scanoss/api/semgrep/__init__.py +++ b/src/scanoss/api/semgrep/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2023, SCANOSS + Copyright (c) 2023, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/semgrep/v2/__init__.py b/src/scanoss/api/semgrep/v2/__init__.py index 31c0cbaa..da9b8e79 100644 --- a/src/scanoss/api/semgrep/v2/__init__.py +++ b/src/scanoss/api/semgrep/v2/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2023, SCANOSS + Copyright (c) 2023, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py index 1b8b0461..a3550b90 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/semgrep/v2/scanoss-semgrep.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,26 +17,29 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12\".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse\"!\x82\xd3\xe4\x93\x02\x1b\"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service\"J\n\x0fscanoss-semgrep\x12\"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse"\x1f\x82\xd3\xe4\x93\x02\x19"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse"!\x82\xd3\xe4\x93\x02\x1b"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service"J\n\x0fscanoss-semgrep\x12"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' +) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.semgrep.v2.scanoss_semgrep_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service\"J\n\017scanoss-semgrep\022\"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _SEMGREP.methods_by_name['Echo']._options = None - _SEMGREP.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\"\024/api/v2/semgrep/echo:\001*' - _SEMGREP.methods_by_name['GetIssues']._options = None - _SEMGREP.methods_by_name['GetIssues']._serialized_options = b'\202\323\344\223\002\033\"\026/api/v2/semgrep/issues:\001*' - _SEMGREPRESPONSE._serialized_start=193 - _SEMGREPRESPONSE._serialized_end=599 - _SEMGREPRESPONSE_ISSUE._serialized_start=329 - _SEMGREPRESPONSE_ISSUE._serialized_end=396 - _SEMGREPRESPONSE_FILE._serialized_start=398 - _SEMGREPRESPONSE_FILE._serialized_end=498 - _SEMGREPRESPONSE_PURLS._serialized_start=500 - _SEMGREPRESPONSE_PURLS._serialized_end=599 - _SEMGREP._serialized_start=602 - _SEMGREP._serialized_end=850 + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service"J\n\017scanoss-semgrep\022"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _SEMGREP.methods_by_name['Echo']._options = None + _SEMGREP.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031"\024/api/v2/semgrep/echo:\001*' + _SEMGREP.methods_by_name['GetIssues']._options = None + _SEMGREP.methods_by_name[ + 'GetIssues' + ]._serialized_options = b'\202\323\344\223\002\033"\026/api/v2/semgrep/issues:\001*' + _SEMGREPRESPONSE._serialized_start = 193 + _SEMGREPRESPONSE._serialized_end = 599 + _SEMGREPRESPONSE_ISSUE._serialized_start = 329 + _SEMGREPRESPONSE_ISSUE._serialized_end = 396 + _SEMGREPRESPONSE_FILE._serialized_start = 398 + _SEMGREPRESPONSE_FILE._serialized_end = 498 + _SEMGREPRESPONSE_PURLS._serialized_start = 500 + _SEMGREPRESPONSE_PURLS._serialized_end = 599 + _SEMGREP._serialized_start = 602 + _SEMGREP._serialized_end = 850 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py index 4748a3ee..d11ba8f0 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py @@ -1,5 +1,6 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" + import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 @@ -18,15 +19,15 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.semgrep.v2.Semgrep/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.semgrep.v2.Semgrep/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.GetIssues = channel.unary_unary( - '/scanoss.api.semgrep.v2.Semgrep/GetIssues', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, - ) + '/scanoss.api.semgrep.v2.Semgrep/GetIssues', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, + ) class SemgrepServicer(object): @@ -35,15 +36,13 @@ class SemgrepServicer(object): """ def Echo(self, request, context): - """Standard echo - """ + """Standard echo""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetIssues(self, request, context): - """Get Potential issues associated with a list of PURLs - """ + """Get Potential issues associated with a list of PURLs""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -51,58 +50,81 @@ def GetIssues(self, request, context): def add_SemgrepServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'GetIssues': grpc.unary_unary_rpc_method_handler( - servicer.GetIssues, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetIssues': grpc.unary_unary_rpc_method_handler( + servicer.GetIssues, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.SerializeToString, + ), } - generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) + generic_handler = grpc.method_handlers_generic_handler('scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) - # This class is part of an EXPERIMENTAL API. +# This class is part of an EXPERIMENTAL API. class Semgrep(object): """ Expose all of the SCANOSS Cryptography RPCs here """ @staticmethod - def Echo(request, + def Echo( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/Echo', + '/scanoss.api.semgrep.v2.Semgrep/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) @staticmethod - def GetIssues(request, + def GetIssues( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/GetIssues', + '/scanoss.api.semgrep.v2.Semgrep/GetIssues', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/src/scanoss/api/vulnerabilities/__init__.py b/src/scanoss/api/vulnerabilities/__init__.py index 0ac8eded..8c08839c 100644 --- a/src/scanoss/api/vulnerabilities/__init__.py +++ b/src/scanoss/api/vulnerabilities/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2022, SCANOSS + Copyright (c) 2022, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/vulnerabilities/v2/__init__.py b/src/scanoss/api/vulnerabilities/v2/__init__.py index 0ac8eded..8c08839c 100644 --- a/src/scanoss/api/vulnerabilities/v2/__init__.py +++ b/src/scanoss/api/vulnerabilities/v2/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2022, SCANOSS + Copyright (c) 2022, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index 9fc87ed3..9b4ea185 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -2,6 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto """Generated protocol buffer code.""" + from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,34 +17,43 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n None: """ Setup all the command line arguments for processing """ - parser = argparse.ArgumentParser(description=f'SCANOSS Python CLI. Ver: {__version__}, License: MIT, Fast Winnowing: {FAST_WINNOWING}') + parser = argparse.ArgumentParser( + description=f'SCANOSS Python CLI. Ver: {__version__}, License: MIT, Fast Winnowing: {FAST_WINNOWING}' + ) parser.add_argument('--version', '-v', action='store_true', help='Display version details') - subparsers = parser.add_subparsers(title='Sub Commands', dest='subparser', description='valid subcommands', - help='sub-command help') + subparsers = parser.add_subparsers( + title='Sub Commands', dest='subparser', description='valid subcommands', help='sub-command help' + ) # Sub-command: version - p_ver = subparsers.add_parser('version', aliases=['ver'], - description=f'Version of SCANOSS CLI: {__version__}', help='SCANOSS version') + p_ver = subparsers.add_parser( + 'version', aliases=['ver'], description=f'Version of SCANOSS CLI: {__version__}', help='SCANOSS version' + ) p_ver.set_defaults(func=ver) # Sub-command: scan - p_scan = subparsers.add_parser('scan', aliases=['sc'], - description=f'Analyse/scan the given source base: {__version__}', - help='Scan source code') + p_scan = subparsers.add_parser( + 'scan', + aliases=['sc'], + description=f'Analyse/scan the given source base: {__version__}', + help='Scan source code', + ) p_scan.set_defaults(func=scan) p_scan.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') - p_scan.add_argument('--wfp', '-w', type=str, - help='Scan a WFP File instead of a folder (optional)') - p_scan.add_argument('--dep', '-p', type=str, - help='Use a dependency file instead of a folder (optional)') - p_scan.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, - help='Scan the file contents supplied via STDIN (optional)') - p_scan.add_argument('--files', '-e', type=str, nargs="*", help='List of files to scan.') + p_scan.add_argument('--wfp', '-w', type=str, help='Scan a WFP File instead of a folder (optional)') + p_scan.add_argument('--dep', '-p', type=str, help='Use a dependency file instead of a folder (optional)') + p_scan.add_argument( + '--stdin', '-s', metavar='STDIN-FILENAME', type=str, help='Scan the file contents supplied via STDIN (optional)' + ) + p_scan.add_argument('--files', '-e', type=str, nargs='*', help='List of files to scan.') p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file') - p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file') - p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') - p_scan.add_argument('--format', '-f', type=str, choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], - help='Result output format (optional - default: plain)') - p_scan.add_argument('--threads', '-T', type=int, default=5, - help='Number of threads to use while scanning (optional - default 5)') - p_scan.add_argument('--flags', '-F', type=int, - help='Scanning engine flags (1: disable snippet matching, 2 enable snippet ids, ' - '4: disable dependencies, 8: disable licenses, 16: disable copyrights,' - '32: disable vulnerabilities, 64: disable quality, 128: disable cryptography,' - '256: disable best match only, 512: hide identified files, ' - '1024: enable download_url, 2048: enable GitHub full path, ' - '4096: disable extended server stats)') - p_scan.add_argument('--post-size', '-P', type=int, default=32, - help='Number of kilobytes to limit the post to while scanning (optional - default 32)') - p_scan.add_argument('--timeout', '-M', type=int, default=180, - help='Timeout (in seconds) for API communication (optional - default 180)') - p_scan.add_argument('--retry', '-R', type=int, default=5, - help='Retry limit for API communication (optional - default 5)') + p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file') + p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + p_scan.add_argument( + '--format', + '-f', + type=str, + choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], + help='Result output format (optional - default: plain)', + ) + p_scan.add_argument( + '--threads', '-T', type=int, default=5, help='Number of threads to use while scanning (optional - default 5)' + ) + p_scan.add_argument( + '--flags', + '-F', + type=int, + help='Scanning engine flags (1: disable snippet matching, 2 enable snippet ids, ' + '4: disable dependencies, 8: disable licenses, 16: disable copyrights,' + '32: disable vulnerabilities, 64: disable quality, 128: disable cryptography,' + '256: disable best match only, 512: hide identified files, ' + '1024: enable download_url, 2048: enable GitHub full path, ' + '4096: disable extended server stats)', + ) + p_scan.add_argument( + '--post-size', + '-P', + type=int, + default=32, + help='Number of kilobytes to limit the post to while scanning (optional - default 32)', + ) + p_scan.add_argument( + '--timeout', + '-M', + type=int, + default=180, + help='Timeout (in seconds) for API communication (optional - default 180)', + ) + p_scan.add_argument( + '--retry', '-R', type=int, default=5, help='Retry limit for API communication (optional - default 5)' + ) p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') - p_scan.add_argument('--sc-command', type=str, - help='Scancode command and path if required (optional - default scancode).') - p_scan.add_argument('--sc-timeout', type=int, default=600, - help='Timeout (in seconds) for scancode to complete (optional - default 600)') - p_scan.add_argument('--dep-scope', '-ds', type=SCOPE, help='Filter dependencies by scope - default all (options: dev/prod)') - p_scan.add_argument('--dep-scope-inc', '-dsi', type=str,help='Include dependencies with declared scopes') + p_scan.add_argument( + '--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' + ) + p_scan.add_argument( + '--sc-timeout', + type=int, + default=600, + help='Timeout (in seconds) for scancode to complete (optional - default 600)', + ) + p_scan.add_argument( + '--dep-scope', '-ds', type=SCOPE, help='Filter dependencies by scope - default all (options: dev/prod)' + ) + p_scan.add_argument('--dep-scope-inc', '-dsi', type=str, help='Include dependencies with declared scopes') p_scan.add_argument('--dep-scope-exc', '-dse', type=str, help='Exclude dependencies with declared scopes') p_scan.add_argument( - '--settings', '-st', + '--settings', + '-st', type=str, help='Settings file to use for scanning (optional - default scanoss.json)', ) p_scan.add_argument( - '--skip-settings-file', '-stf', action='store_true', + '--skip-settings-file', + '-stf', + action='store_true', help='Skip default settings file (scanoss.json) if it exists', ) # Sub-command: fingerprint - p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'], - description=f'Fingerprint the given source base: {__version__}', - help='Fingerprint source code') + p_wfp = subparsers.add_parser( + 'fingerprint', + aliases=['fp', 'wfp'], + description=f'Fingerprint the given source base: {__version__}', + help='Fingerprint source code', + ) p_wfp.set_defaults(func=wfp) - p_wfp.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', - help='A file or folder to scan') - p_wfp.add_argument('--stdin', '-s', metavar='STDIN-FILENAME', type=str, - help='Fingerprint the file contents supplied via STDIN (optional)') + p_wfp.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') + p_wfp.add_argument( + '--stdin', + '-s', + metavar='STDIN-FILENAME', + type=str, + help='Fingerprint the file contents supplied via STDIN (optional)', + ) p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_wfp.add_argument( - '--settings', '-st', + '--settings', + '-st', type=str, help='Settings file to use for fingerprinting (optional - default scanoss.json)', ) p_wfp.add_argument( - '--skip-settings-file', '-stf', action='store_true', + '--skip-settings-file', + '-stf', + action='store_true', help='Skip default settings file (scanoss.json) if it exists', ) # Sub-command: dependency - p_dep = subparsers.add_parser('dependencies', aliases=['dp', 'dep'], - description=f'Produce dependency file summary: {__version__}', - help='Scan source code for dependencies, but do not decorate them') + p_dep = subparsers.add_parser( + 'dependencies', + aliases=['dp', 'dep'], + description=f'Produce dependency file summary: {__version__}', + help='Scan source code for dependencies, but do not decorate them', + ) p_dep.set_defaults(func=dependency) p_dep.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_dep.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') - p_dep.add_argument('--sc-command', type=str, - help='Scancode command and path if required (optional - default scancode).') - p_dep.add_argument('--sc-timeout', type=int, default=600, - help='Timeout (in seconds) for scancode to complete (optional - default 600)') + p_dep.add_argument( + '--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' + ) + p_dep.add_argument( + '--sc-timeout', + type=int, + default=600, + help='Timeout (in seconds) for scancode to complete (optional - default 600)', + ) # Sub-command: file_count - p_fc = subparsers.add_parser('file_count', aliases=['fc'], - description=f'Produce a file type count summary: {__version__}', - help='Search the source tree and produce a file type summary') + p_fc = subparsers.add_parser( + 'file_count', + aliases=['fc'], + description=f'Produce a file type count summary: {__version__}', + help='Search the source tree and produce a file type summary', + ) p_fc.set_defaults(func=file_count) p_fc.add_argument('scan_dir', metavar='DIR', type=str, nargs='?', help='A folder to search') p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_fc.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') # Sub-command: convert - p_cnv = subparsers.add_parser('convert', aliases=['cv', 'cnv', 'cvrt'], - description=f'Convert results files between formats: {__version__}', - help='Convert file format') + p_cnv = subparsers.add_parser( + 'convert', + aliases=['cv', 'cnv', 'cvrt'], + description=f'Convert results files between formats: {__version__}', + help='Convert file format', + ) p_cnv.set_defaults(func=convert) p_cnv.add_argument('--input', '-i', type=str, required=True, help='Input file name') p_cnv.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') - p_cnv.add_argument('--format', '-f', type=str, choices=['cyclonedx', 'spdxlite', 'csv'], default='spdxlite', - help='Output format (optional - default: spdxlite)') - p_cnv.add_argument('--input-format', type=str, choices=['plain'], default='plain', - help='Input format (optional - default: plain)') + p_cnv.add_argument( + '--format', + '-f', + type=str, + choices=['cyclonedx', 'spdxlite', 'csv'], + default='spdxlite', + help='Output format (optional - default: spdxlite)', + ) + p_cnv.add_argument( + '--input-format', type=str, choices=['plain'], default='plain', help='Input format (optional - default: plain)' + ) # Sub-command: component - p_comp = subparsers.add_parser('component', aliases=['comp'], - description=f'SCANOSS Component commands: {__version__}', - help='Component support commands') + p_comp = subparsers.add_parser( + 'component', + aliases=['comp'], + description=f'SCANOSS Component commands: {__version__}', + help='Component support commands', + ) - comp_sub = p_comp.add_subparsers(title='Component Commands', dest='subparsercmd', description='component sub-commands', - help='component sub-commands') + comp_sub = p_comp.add_subparsers( + title='Component Commands', + dest='subparsercmd', + description='component sub-commands', + help='component sub-commands', + ) # Component Sub-command: component crypto - c_crypto = comp_sub.add_parser('crypto', aliases=['cr'], - description=f'Show Cryptographic algorithms: {__version__}', - help='Retrieve cryptographic algorithms for the given components') + c_crypto = comp_sub.add_parser( + 'crypto', + aliases=['cr'], + description=f'Show Cryptographic algorithms: {__version__}', + help='Retrieve cryptographic algorithms for the given components', + ) c_crypto.set_defaults(func=comp_crypto) # Component Sub-command: component vulns - c_vulns = comp_sub.add_parser('vulns', aliases=['vulnerabilities', 'vu'], - description=f'Show Vulnerability details: {__version__}', - help='Retrieve vulnerabilities for the given components') + c_vulns = comp_sub.add_parser( + 'vulns', + aliases=['vulnerabilities', 'vu'], + description=f'Show Vulnerability details: {__version__}', + help='Retrieve vulnerabilities for the given components', + ) c_vulns.set_defaults(func=comp_vulns) # Component Sub-command: component semgrep - c_semgrep = comp_sub.add_parser('semgrep', aliases=['sp'], - description=f'Show Semgrep findings: {__version__}', - help='Retrieve semgrep issues/findings for the given components') + c_semgrep = comp_sub.add_parser( + 'semgrep', + aliases=['sp'], + description=f'Show Semgrep findings: {__version__}', + help='Retrieve semgrep issues/findings for the given components', + ) c_semgrep.set_defaults(func=comp_semgrep) # Component Sub-command: component provenance @@ -206,9 +291,12 @@ def setup_args() -> None: # Component Sub-command: component search - c_search = comp_sub.add_parser('search', aliases=['sc'], - description=f'Search component details: {__version__}', - help='Search for a KB component') + c_search = comp_sub.add_parser( + 'search', + aliases=['sc'], + description=f'Search component details: {__version__}', + help='Search for a KB component', + ) c_search.add_argument('--input', '-i', type=str, help='Input file name') c_search.add_argument('--search', '-s', type=str, help='Generic component search') c_search.add_argument('--vendor', '-v', type=str, help='Generic component search') @@ -219,68 +307,100 @@ def setup_args() -> None: c_search.set_defaults(func=comp_search) # Component Sub-command: component versions - c_versions = comp_sub.add_parser('versions', aliases=['vs'], - description=f'Get component version details: {__version__}', - help='Search for component versions') + c_versions = comp_sub.add_parser( + 'versions', + aliases=['vs'], + description=f'Get component version details: {__version__}', + help='Search for component versions', + ) c_versions.add_argument('--input', '-i', type=str, help='Input file name') c_versions.add_argument('--purl', '-p', type=str, help='Generic component search') c_versions.add_argument('--limit', '-l', type=int, help='Generic component search') c_versions.set_defaults(func=comp_versions) # Common purl Component sub-command options - for p in [c_crypto, c_vulns, c_semgrep, c_provenance]: - p.add_argument('--purl', '-p', type=str, nargs="*", help='Package URL - PURL to process.') + for p in [c_crypto, c_vulns, c_semgrep]: + p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') # Common Component sub-command options for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') - p.add_argument('--timeout', '-M', type=int, default=600, - help='Timeout (in seconds) for API communication (optional - default 600)') + p.add_argument( + '--timeout', + '-M', + type=int, + default=600, + help='Timeout (in seconds) for API communication (optional - default 600)', + ) # Sub-command: utils - p_util = subparsers.add_parser('utils', aliases=['ut'], - description=f'SCANOSS Utility commands: {__version__}', - help='General utility support commands') + p_util = subparsers.add_parser( + 'utils', + aliases=['ut'], + description=f'SCANOSS Utility commands: {__version__}', + help='General utility support commands', + ) - utils_sub = p_util.add_subparsers(title='Utils Commands', dest='subparsercmd', description='utils sub-commands', - help='utils sub-commands') + utils_sub = p_util.add_subparsers( + title='Utils Commands', dest='subparsercmd', description='utils sub-commands', help='utils sub-commands' + ) # Utils Sub-command: utils fast - p_f_f = utils_sub.add_parser('fast', - description=f'Is fast winnowing enabled: {__version__}', help='SCANOSS fast winnowing') + p_f_f = utils_sub.add_parser( + 'fast', description=f'Is fast winnowing enabled: {__version__}', help='SCANOSS fast winnowing' + ) p_f_f.set_defaults(func=fast) # Utils Sub-command: utils certloc - p_c_loc = utils_sub.add_parser('certloc', aliases=['cl'], - description=f'Show location of Python CA Certs: {__version__}', - help='Display the location of Python CA Certs') + p_c_loc = utils_sub.add_parser( + 'certloc', + aliases=['cl'], + description=f'Show location of Python CA Certs: {__version__}', + help='Display the location of Python CA Certs', + ) p_c_loc.set_defaults(func=utils_certloc) # Utils Sub-command: utils cert-download - p_c_dwnld = utils_sub.add_parser('cert-download', aliases=['cdl', 'cert-dl'], - description=f'Download Server SSL Cert: {__version__}', - help='Download the specified server\'s SSL PEM certificate') + p_c_dwnld = utils_sub.add_parser( + 'cert-download', + aliases=['cdl', 'cert-dl'], + description=f'Download Server SSL Cert: {__version__}', + help="Download the specified server's SSL PEM certificate", + ) p_c_dwnld.set_defaults(func=utils_cert_download) p_c_dwnld.add_argument('--hostname', '-n', required=True, type=str, help='Server hostname to download cert from.') - p_c_dwnld.add_argument('--port', '-p', required=False, type=int, default=443, - help='Server port number (default: 443).') + p_c_dwnld.add_argument( + '--port', '-p', required=False, type=int, default=443, help='Server port number (default: 443).' + ) p_c_dwnld.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') # Utils Sub-command: utils pac-proxy - p_p_proxy = utils_sub.add_parser('pac-proxy', aliases=['pac'], - description=f'Determine Proxy from PAC: {__version__}', - help='Use Proxy Auto-Config to determine proxy configuration') + p_p_proxy = utils_sub.add_parser( + 'pac-proxy', + aliases=['pac'], + description=f'Determine Proxy from PAC: {__version__}', + help='Use Proxy Auto-Config to determine proxy configuration', + ) p_p_proxy.set_defaults(func=utils_pac_proxy) - p_p_proxy.add_argument('--pac', required=False, type=str, default="auto", - help='Proxy auto configuration. Specify a file, http url or "auto" to try to discover it.' - ) - p_p_proxy.add_argument('--url', required=False, type=str, default="https://api.osskb.org", - help='URL to test (default: https://api.osskb.org).') + p_p_proxy.add_argument( + '--pac', + required=False, + type=str, + default='auto', + help='Proxy auto configuration. Specify a file, http url or "auto" to try to discover it.', + ) + p_p_proxy.add_argument( + '--url', + required=False, + type=str, + default='https://api.osskb.org', + help='URL to test (default: https://api.osskb.org).', + ) p_results = subparsers.add_parser( 'results', aliases=['res'], - description=f"SCANOSS Results commands: {__version__}", + description=f'SCANOSS Results commands: {__version__}', help='Process scan results', ) p_results.add_argument( @@ -318,37 +438,66 @@ def setup_args() -> None: ) p_results.set_defaults(func=results) - # Sub-command: inspect - p_inspect = subparsers.add_parser('inspect', aliases=['insp', 'ins'], - description=f'Inspect results: {__version__}', - help='Inspect results') + p_inspect = subparsers.add_parser( + 'inspect', aliases=['insp', 'ins'], description=f'Inspect results: {__version__}', help='Inspect results' + ) # Sub-parser: inspect - p_inspect_sub = p_inspect.add_subparsers(title='Inspect Commands', dest='subparsercmd', - description='Inspect sub-commands', help='Inspect sub-commands') + p_inspect_sub = p_inspect.add_subparsers( + title='Inspect Commands', dest='subparsercmd', description='Inspect sub-commands', help='Inspect sub-commands' + ) # Inspect Sub-command: inspect copyleft - p_copyleft = p_inspect_sub.add_parser('copyleft', aliases=['cp'],description="Inspect for copyleft licenses", help='Inspect for copyleft licenses') - p_copyleft.add_argument('--include', help='List of Copyleft licenses to append to the default list. Provide licenses as a comma-separated list.') - p_copyleft.add_argument('--exclude', help='List of Copyleft licenses to remove from default list. Provide licenses as a comma-separated list.') - p_copyleft.add_argument('--explicit', help='Explicit list of Copyleft licenses to consider. Provide licenses as a comma-separated list.s') + p_copyleft = p_inspect_sub.add_parser( + 'copyleft', aliases=['cp'], description='Inspect for copyleft licenses', help='Inspect for copyleft licenses' + ) + p_copyleft.add_argument( + '--include', + help='List of Copyleft licenses to append to the default list. Provide licenses as a comma-separated list.', + ) + p_copyleft.add_argument( + '--exclude', + help='List of Copyleft licenses to remove from default list. Provide licenses as a comma-separated list.', + ) + p_copyleft.add_argument( + '--explicit', + help='Explicit list of Copyleft licenses to consider. Provide licenses as a comma-separated list.s', + ) p_copyleft.set_defaults(func=inspect_copyleft) # Inspect Sub-command: inspect undeclared - p_undeclared = p_inspect_sub.add_parser('undeclared', aliases=['un'],description="Inspect for undeclared components", help='Inspect for undeclared components') - p_undeclared.add_argument('--sbom-format',required=False ,choices=['legacy', 'settings'], - default="settings",help='Sbom format for status output') + p_undeclared = p_inspect_sub.add_parser( + 'undeclared', + aliases=['un'], + description='Inspect for undeclared components', + help='Inspect for undeclared components', + ) + p_undeclared.add_argument( + '--sbom-format', + required=False, + choices=['legacy', 'settings'], + default='settings', + help='Sbom format for status output', + ) p_undeclared.set_defaults(func=inspect_undeclared) for p in [p_copyleft, p_undeclared]: p.add_argument('-i', '--input', nargs='?', help='Path to results file') - p.add_argument('-f', '--format',required=False ,choices=['json', 'md', 'jira_md'], default='json', help='Output format (default: json)') + p.add_argument( + '-f', + '--format', + required=False, + choices=['json', 'md', 'jira_md'], + default='json', + help='Output format (default: json)', + ) p.add_argument('-o', '--output', type=str, help='Save details into a file') p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file') # Global Scan command options for p in [p_scan]: - p.add_argument('--apiurl', type=str, - help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)') + p.add_argument( + '--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)' + ) p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') # Global Scan/Fingerprint filter options @@ -361,36 +510,74 @@ def setup_args() -> None: p.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') p.add_argument('--skip-extension', '-E', type=str, action='append', help='File Extension to skip.') p.add_argument('--skip-folder', '-O', type=str, action='append', help='Folder to skip.') - p.add_argument('--skip-size', '-Z', type=int, default=0, - help='Minimum file size to consider for fingerprinting (optional - default 0 bytes [unlimited])') + p.add_argument( + '--skip-size', + '-Z', + type=int, + default=0, + help='Minimum file size to consider for fingerprinting (optional - default 0 bytes [unlimited])', + ) p.add_argument('--skip-md5', '-5', type=str, action='append', help='Skip files matching MD5.') p.add_argument('--strip-hpsm', '-G', type=str, action='append', help='Strip HPSM string from WFP.') p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.') # Global Scan/GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: - p.add_argument('--key', '-k', type=str, - help='SCANOSS API Key token (optional - not required for default OSSKB URL)') - p.add_argument('--proxy', type=str, help='Proxy URL to use for connections (optional). ' - 'Can also use the environment variable "HTTPS_PROXY=:" ' - 'and "grcp_proxy=:" for gRPC') - p.add_argument('--pac', type=str, help='Proxy auto configuration (optional). ' - 'Specify a file, http url or "auto" to try to discover it.') - p.add_argument('--ca-cert', type=str, help='Alternative certificate PEM file (optional). ' - 'Can also use the environment variable ' - '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem" and ' - '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC') + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: + p.add_argument( + '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' + ) + p.add_argument( + '--proxy', + type=str, + help='Proxy URL to use for connections (optional). ' + 'Can also use the environment variable "HTTPS_PROXY=:" ' + 'and "grcp_proxy=:" for gRPC', + ) + p.add_argument( + '--pac', + type=str, + help='Proxy auto configuration (optional). Specify a file, http url or "auto" to try to discover it.', + ) + p.add_argument( + '--ca-cert', + type=str, + help='Alternative certificate PEM file (optional). ' + 'Can also use the environment variable ' + '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem" and ' + '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC', + ) # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: - p.add_argument('--api2url', type=str, - help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)') - p.add_argument('--grpc-proxy', type=str, help='GRPC Proxy URL to use for connections (optional). ' - 'Can also use the environment variable "grcp_proxy=:"') + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: + p.add_argument( + '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' + ) + p.add_argument( + '--grpc-proxy', + type=str, + help='GRPC Proxy URL to use for connections (optional). ' + 'Can also use the environment variable "grcp_proxy=:"', + ) # Help/Trace command options - for p in [p_scan, p_wfp, p_dep, p_fc, p_cnv, p_c_loc, p_c_dwnld, p_p_proxy, c_crypto, c_vulns, c_search, - c_versions, c_semgrep, p_results, p_undeclared, p_copyleft, c_provenance]: + for p in [ + p_scan, + p_wfp, + p_dep, + p_fc, + p_cnv, + p_c_loc, + p_c_dwnld, + p_p_proxy, + c_crypto, + c_vulns, + c_search, + c_versions, + c_semgrep, + p_results, + p_undeclared, + p_copyleft, + ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -403,10 +590,15 @@ def setup_args() -> None: parser.print_help() # No sub command subcommand, print general help exit(1) else: - if ((args.subparser == 'utils' or args.subparser == 'ut' or - args.subparser == 'component' or args.subparser == 'comp' or - args.subparser == 'inspect' or args.subparser == 'insp' or args.subparser == 'ins') - and not args.subparsercmd): + if ( + args.subparser == 'utils' + or args.subparser == 'ut' + or args.subparser == 'component' + or args.subparser == 'comp' + or args.subparser == 'inspect' + or args.subparser == 'insp' + or args.subparser == 'ins' + ) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed exit(1) args.func(parser, args) # Execute the function associated with the sub-command @@ -447,9 +639,13 @@ def file_count(parser, args): scan_output = args.output open(scan_output, 'w').close() - counter = FileCount(debug=args.debug, quiet=args.quiet, trace=args.trace, scan_output=scan_output, - hidden_files_folders=args.all_hidden - ) + counter = FileCount( + debug=args.debug, + quiet=args.quiet, + trace=args.trace, + scan_output=scan_output, + hidden_files_folders=args.all_hidden, + ) if not os.path.exists(args.scan_dir): print_stderr(f'Error: Folder specified does not exist: {args.scan_dir}.') exit(1) @@ -492,13 +688,24 @@ def wfp(parser, args): exit(1) scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value # Skip snippet generation or not - scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, obfuscate=args.obfuscate, - scan_options=scan_options, all_extensions=args.all_extensions, - all_folders=args.all_folders, hidden_files_folders=args.all_hidden, hpsm=args.hpsm, - skip_size=args.skip_size, skip_extensions=args.skip_extension, skip_folders=args.skip_folder, - skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, - scan_settings=scan_settings - ) + scanner = Scanner( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + obfuscate=args.obfuscate, + scan_options=scan_options, + all_extensions=args.all_extensions, + all_folders=args.all_folders, + hidden_files_folders=args.all_hidden, + hpsm=args.hpsm, + skip_size=args.skip_size, + skip_extensions=args.skip_extension, + skip_folders=args.skip_folder, + skip_md5_ids=args.skip_md5, + strip_hpsm_ids=args.strip_hpsm, + strip_snippet_ids=args.strip_snippet, + scan_settings=scan_settings, + ) if args.stdin: contents = sys.stdin.buffer.read() scanner.wfp_contents(args.stdin, contents, scan_output) @@ -582,11 +789,17 @@ def scan(parser, args): scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) try: if args.identify: - scan_settings.load_json_file(args.identify, args.scan_dir).set_file_type('legacy').set_scan_type('identify') + scan_settings.load_json_file(args.identify, args.scan_dir).set_file_type('legacy').set_scan_type( + 'identify' + ) elif args.ignore: - scan_settings.load_json_file(args.ignore, args.scan_dir).set_file_type('legacy').set_scan_type('blacklist') + scan_settings.load_json_file(args.ignore, args.scan_dir).set_file_type('legacy').set_scan_type( + 'blacklist' + ) else: - scan_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new').set_scan_type('identify') + scan_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new').set_scan_type( + 'identify' + ) except ScanossSettingsError as e: print_stderr(f'Error: {e}') exit(1) @@ -611,13 +824,13 @@ def scan(parser, args): if args.skip_settings_file: print_stderr('Skipping Settings file...') if args.all_extensions: - print_stderr("Scanning all file extensions/types...") + print_stderr('Scanning all file extensions/types...') if args.all_folders: - print_stderr("Scanning all folders...") + print_stderr('Scanning all folders...') if args.all_hidden: - print_stderr("Scanning all hidden files/folders...") + print_stderr('Scanning all hidden files/folders...') if args.skip_snippets: - print_stderr("Skipping snippets...") + print_stderr('Skipping snippets...') if args.post_size != 32: print_stderr(f'Changing scanning POST size to: {args.post_size}k...') if args.timeout != 180: @@ -625,7 +838,7 @@ def scan(parser, args): if args.retry != 5: print_stderr(f'Changing scanning POST retry to: {args.retry}...') if args.obfuscate: - print_stderr("Obfuscating file fingerprints...") + print_stderr('Obfuscating file fingerprints...') if args.proxy: print_stderr(f'Using Proxy {args.proxy}...') if args.grpc_proxy: @@ -635,7 +848,7 @@ def scan(parser, args): if args.ca_cert: print_stderr(f'Using Certificate {args.ca_cert}...') if args.hpsm: - print_stderr("Setting HPSM mode...") + print_stderr('Setting HPSM mode...') if flags: print_stderr(f'Using flags {flags}...') elif not args.quiet: @@ -654,8 +867,11 @@ def scan(parser, args): scan_options = get_scan_options(args) # Figure out what scanning options we have scanner = Scanner( - debug=args.debug, trace=args.trace, quiet=args.quiet, - api_key=args.key, url=args.apiurl, + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + api_key=args.key, + url=args.apiurl, scan_output=scan_output, output_format=output_format, flags=flags, @@ -699,21 +915,31 @@ def scan(parser, args): if not scanner.scan_contents(args.stdin, contents): exit(1) elif args.files: - if not scanner.scan_files_with_options( - args.files, args.dep, scanner.winnowing.file_map - ): + if not scanner.scan_files_with_options(args.files, args.dep, scanner.winnowing.file_map): exit(1) elif args.scan_dir: if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') exit(1) if os.path.isdir(args.scan_dir): - if not scanner.scan_folder_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map, - args.dep_scope, args.dep_scope_inc, args.dep_scope_exc): + if not scanner.scan_folder_with_options( + args.scan_dir, + args.dep, + scanner.winnowing.file_map, + args.dep_scope, + args.dep_scope_inc, + args.dep_scope_exc, + ): exit(1) elif os.path.isfile(args.scan_dir): - if not scanner.scan_file_with_options(args.scan_dir, args.dep, scanner.winnowing.file_map, - args.dep_scope, args.dep_scope_inc, args.dep_scope_exc): + if not scanner.scan_file_with_options( + args.scan_dir, + args.dep, + scanner.winnowing.file_map, + args.dep_scope, + args.dep_scope_inc, + args.dep_scope_exc, + ): exit(1) else: print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') @@ -724,8 +950,9 @@ def scan(parser, args): f'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.' ) exit(1) - if not scanner.scan_folder_with_options(".", args.dep, scanner.winnowing.file_map,args.dep_scope, - args.dep_scope_inc, args.dep_scope_exc): + if not scanner.scan_folder_with_options( + '.', args.dep, scanner.winnowing.file_map, args.dep_scope, args.dep_scope_inc, args.dep_scope_exc + ): exit(1) else: print_stderr('No action found to process') @@ -754,9 +981,9 @@ def dependency(parser, args): scan_output = args.output open(scan_output, 'w').close() - sc_deps = ScancodeDeps(debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, - timeout=args.sc_timeout - ) + sc_deps = ScancodeDeps( + debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout + ) if not sc_deps.get_dependencies(what_to_scan=args.scan_dir, result_output=scan_output): exit(1) @@ -796,6 +1023,7 @@ def convert(parser, args): if not success: exit(1) + def inspect_copyleft(parser, args): """ Run the "inspect" sub-command @@ -820,12 +1048,22 @@ def inspect_copyleft(parser, args): status_output = args.status open(status_output, 'w').close() - i_copyleft = Copyleft(debug=args.debug, trace=args.trace, quiet=args.quiet, filepath=args.input, - format_type=args.format, status=status_output, output=output, include=args.include, - exclude=args.exclude, explicit=args.explicit) + i_copyleft = Copyleft( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + filepath=args.input, + format_type=args.format, + status=status_output, + output=output, + include=args.include, + exclude=args.exclude, + explicit=args.explicit, + ) status, _ = i_copyleft.run() sys.exit(status) + def inspect_undeclared(parser, args): """ Run the "inspect" sub-command @@ -849,20 +1087,30 @@ def inspect_undeclared(parser, args): if args.status: status_output = args.status open(status_output, 'w').close() - i_undeclared = UndeclaredComponent(debug=args.debug, trace=args.trace, quiet=args.quiet, - filepath=args.input, format_type=args.format, - status=status_output, output=output, sbom_format=args.sbom_format) + i_undeclared = UndeclaredComponent( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + filepath=args.input, + format_type=args.format, + status=status_output, + output=output, + sbom_format=args.sbom_format, + ) status, _ = i_undeclared.run() sys.exit(status) + def utils_certloc(*_): """ Run the "utils certloc" sub-command :param _: ignored/unused """ import certifi + print(f'CA Cert File: {certifi.where()}') + def utils_cert_download(_, args): """ Run the "utils cert-download" sub-command @@ -897,7 +1145,9 @@ def utils_cert_download(_, args): if not args.quiet: print_stderr(f'Certificate {index} - CN: {cn}') if sys.version_info[0] >= 3: - print((crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file) # Print the downloaded PEM certificate + print( + (crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file + ) # Print the downloaded PEM certificate else: print((crypto.dump_certificate(crypto.FILETYPE_PEM, cert)).strip(), file=file) except SSL.Error as e: @@ -919,6 +1169,7 @@ def utils_pac_proxy(_, args): :param args: Parsed arguments """ from pypac.resolver import ProxyResolver + if not args.pac: print_stderr(f'Error: No pac file option specified.') exit(1) @@ -974,9 +1225,18 @@ def comp_crypto(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) pac_file = get_pac_file(args.pac) - comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout) + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.api2url, + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + timeout=args.timeout, + ) if not comps.get_crypto_details(args.input, args.purl, args.output): exit(1) @@ -999,12 +1259,22 @@ def comp_vulns(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) pac_file = get_pac_file(args.pac) - comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout) + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.api2url, + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + timeout=args.timeout, + ) if not comps.get_vulnerabilities(args.input, args.purl, args.output): exit(1) + def comp_semgrep(parser, args): """ Run the "component semgrep" sub-command @@ -1023,35 +1293,21 @@ def comp_semgrep(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) pac_file = get_pac_file(args.pac) - comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout) + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.api2url, + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + timeout=args.timeout, + ) if not comps.get_semgrep_details(args.input, args.purl, args.output): exit(1) -def comp_provenance(parser, args): - """ - Run the "component provenance" sub-command - Parameters - ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments - """ - if (not args.purl and not args.input) or (args.purl and args.input): - print_stderr('Please specify an input file or purl to decorate (--purl or --input)') - parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) - if args.ca_cert and not os.path.exists(args.ca_cert): - print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') - exit(1) - pac_file = get_pac_file(args.pac) - comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout) - if not comps.get_provenance_details(args.input, args.purl, args.output): - exit(1) def comp_search(parser, args): """ @@ -1063,8 +1319,9 @@ def comp_search(parser, args): args: Namespace Parsed arguments """ - if ((not args.input and not args.search and not args.vendor and not args.comp) or - (args.input and (args.search or args.vendor or args.comp))): + if (not args.input and not args.search and not args.vendor and not args.comp) or ( + args.input and (args.search or args.vendor or args.comp) + ): print_stderr('Please specify an input file or search terms (--input or --search, or --vendor or --comp.)') parser.parse_args([args.subparser, args.subparsercmd, '-h']) exit(1) @@ -1073,12 +1330,28 @@ def comp_search(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) pac_file = get_pac_file(args.pac) - comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout) - if not comps.search_components(args.output, json_file=args.input, - search=args.search, vendor=args.vendor, comp=args.comp, package=args.package, - limit=args.limit, offset=args.offset): + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.api2url, + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + timeout=args.timeout, + ) + if not comps.search_components( + args.output, + json_file=args.input, + search=args.search, + vendor=args.vendor, + comp=args.comp, + package=args.package, + limit=args.limit, + offset=args.offset, + ): exit(1) @@ -1101,9 +1374,18 @@ def comp_versions(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') exit(1) pac_file = get_pac_file(args.pac) - comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout) + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.api2url, + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + timeout=args.timeout, + ) if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit): exit(1) @@ -1120,13 +1402,13 @@ def results(parser, args): """ if not args.filepath: print_stderr('ERROR: Please specify a file containing the results') - parser.parse_args([args.subparser, "-h"]) + parser.parse_args([args.subparser, '-h']) exit(1) file_path = Path(args.filepath).resolve() if not file_path.is_file(): - print_stderr(f"The specified file {args.filepath} does not exist") + print_stderr(f'The specified file {args.filepath} does not exist') exit(1) results = Results( @@ -1155,5 +1437,5 @@ def main(): setup_args() -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 37ebfefa..e98e7a80 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2023, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2023, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os import sys @@ -38,10 +39,19 @@ class Components(ScanossBase): Class for Component functionality """ - def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False, - grpc_url: str = None, api_key: str = None, timeout: int = 600, - proxy: str = None, grpc_proxy: str = None, ca_cert: str = None, pac: PACFile = None - ): + def __init__( + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + grpc_url: str = None, + api_key: str = None, + timeout: int = 600, + proxy: str = None, + grpc_proxy: str = None, + ca_cert: str = None, + pac: PACFile = None, + ): """ Handle all component style requests @@ -58,9 +68,19 @@ def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False """ super().__init__(debug, trace, quiet) ver_details = Scanner.version_details() - self.grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, - ver_details=ver_details, ca_cert=ca_cert, proxy=proxy, pac=pac, - grpc_proxy=grpc_proxy, timeout=timeout) + self.grpc_api = ScanossGrpc( + url=grpc_url, + debug=debug, + quiet=quiet, + trace=trace, + api_key=api_key, + ver_details=ver_details, + ca_cert=ca_cert, + proxy=proxy, + pac=pac, + grpc_proxy=grpc_proxy, + timeout=timeout, + ) def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None) -> Optional[dict]: """ @@ -222,9 +242,17 @@ def get_semgrep_details(self, json_file: str = None, purls: [] = None, output_fi self._close_file(output_file, file) return success - def search_components(self, output_file: str = None, json_file: str = None, - search: str = None, vendor: str = None, comp: str = None, package: str = None, - limit: int = None, offset: int = None) -> bool: + def search_components( + self, + output_file: str = None, + json_file: str = None, + search: str = None, + vendor: str = None, + comp: str = None, + package: str = None, + limit: int = None, + offset: int = None, + ) -> bool: """ Search for a component based on the given search criteria @@ -245,16 +273,11 @@ def search_components(self, output_file: str = None, json_file: str = None, if request is None: return False else: # Construct a query dictionary from parameters - request = { - "search": search, - "vendor": vendor, - "component": comp, - "package": package - } + request = {'search': search, 'vendor': vendor, 'component': comp, 'package': package} if limit is not None and limit > 0: - request["limit"] = limit + request['limit'] = limit if offset is not None and offset > 0: - request["offset"] = offset + request['offset'] = offset file = self._open_file_or_sdtout(output_file) if file is None: @@ -269,8 +292,9 @@ def search_components(self, output_file: str = None, json_file: str = None, self._close_file(output_file, file) return success - def get_component_versions(self, output_file: str = None, json_file: str = None, - purl: str = None, limit: int = None) -> bool: + def get_component_versions( + self, output_file: str = None, json_file: str = None, purl: str = None, limit: int = None + ) -> bool: """ Search for a component versions based on the given search criteria @@ -287,11 +311,9 @@ def get_component_versions(self, output_file: str = None, json_file: str = None, if request is None: return False else: # Construct a query dictionary from parameters - request = { - "purl": purl - } + request = {'purl': purl} if limit is not None and limit > 0: - request["limit"] = limit + request['limit'] = limit file = self._open_file_or_sdtout(output_file) if file is None: diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index cda12d1a..5b36284c 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2022, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os.path import sys @@ -59,21 +60,21 @@ def parse(self, data: json): file_details = data.get(f) # print(f'File: {f}: {file_details}') for d in file_details: - id_details = d.get("id") + id_details = d.get('id') if not id_details or id_details == 'none': continue - matched = d.get("matched", '') - lines = d.get("lines", '').replace(',', ';') # swap comma with semicolon to help basic parsers - oss_lines = d.get("oss_lines", '').replace(',', ';') + matched = d.get('matched', '') + lines = d.get('lines', '').replace(',', ';') # swap comma with semicolon to help basic parsers + oss_lines = d.get('oss_lines', '').replace(',', ';') detected = {} if id_details == 'dependency': - dependencies = d.get("dependencies") + dependencies = d.get('dependencies') if not dependencies: self.print_stderr(f'Warning: No Dependencies found for {f}: {file_details}') continue for deps in dependencies: detected = {} - purl = deps.get("purl") + purl = deps.get('purl') if not purl: self.print_stderr(f'Warning: No PURL found for {f}: {deps}') continue @@ -84,7 +85,7 @@ def parse(self, data: json): dc = [] if licenses: for lic in licenses: - name = lic.get("name") + name = lic.get('name') if name and name not in dc: # Only save the license name once dc.append(name) if not dc or len(dc) == 0: @@ -92,17 +93,23 @@ def parse(self, data: json): else: detected['licenses'] = ';'.join(dc) # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl - csv_dict.append({'inventory_id': row_id, 'path': f, 'detected_usage': id_details, - 'detected_component': detected.get('component'), - 'detected_license': detected.get('licenses'), - 'detected_version': detected.get('version'), - 'detected_latest': detected.get('latest'), - 'detected_purls': detected.get('purls'), - 'detected_url': detected.get('url'), - 'detected_path': detected.get('file', ''), - 'detected_match': matched, 'detected_lines': lines, - 'detected_oss_lines': oss_lines - }) + csv_dict.append( + { + 'inventory_id': row_id, + 'path': f, + 'detected_usage': id_details, + 'detected_component': detected.get('component'), + 'detected_license': detected.get('licenses'), + 'detected_version': detected.get('version'), + 'detected_latest': detected.get('latest'), + 'detected_purls': detected.get('purls'), + 'detected_url': detected.get('url'), + 'detected_path': detected.get('file', ''), + 'detected_match': matched, + 'detected_lines': lines, + 'detected_oss_lines': oss_lines, + } + ) row_id = row_id + 1 else: purls = d.get('purl') @@ -123,25 +130,31 @@ def parse(self, data: json): dc = [] if licenses: for lic in licenses: - name = lic.get("name") + name = lic.get('name') if name and name not in dc: # Only save the license name once - dc.append(lic.get("name")) + dc.append(lic.get('name')) if not dc or len(dc) == 0: detected['licenses'] = '' else: detected['licenses'] = ';'.join(dc) # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl - csv_dict.append({'inventory_id': row_id, 'path': f, 'detected_usage': id_details, - 'detected_component': detected.get('component'), - 'detected_license': detected.get('licenses'), - 'detected_version': detected.get('version'), - 'detected_latest': detected.get('latest'), - 'detected_purls': detected.get('purls'), - 'detected_url': detected.get('url'), - 'detected_path': detected.get('file', ''), - 'detected_match': matched, 'detected_lines': lines, - 'detected_oss_lines': oss_lines - }) + csv_dict.append( + { + 'inventory_id': row_id, + 'path': f, + 'detected_usage': id_details, + 'detected_component': detected.get('component'), + 'detected_license': detected.get('licenses'), + 'detected_version': detected.get('version'), + 'detected_latest': detected.get('latest'), + 'detected_purls': detected.get('purls'), + 'detected_url': detected.get('url'), + 'detected_path': detected.get('file', ''), + 'detected_match': matched, + 'detected_lines': lines, + 'detected_oss_lines': oss_lines, + } + ) row_id = row_id + 1 return csv_dict @@ -174,16 +187,28 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: self.print_stderr('ERROR: No CSV data returned for the JSON string provided.') return False # Header row/column details - fields = ['inventory_id', 'path', 'detected_usage', 'detected_component', 'detected_license', - 'detected_version', 'detected_latest', 'detected_purls', 'detected_url', 'detected_match', - 'detected_lines', 'detected_oss_lines', 'detected_path'] + fields = [ + 'inventory_id', + 'path', + 'detected_usage', + 'detected_component', + 'detected_license', + 'detected_version', + 'detected_latest', + 'detected_purls', + 'detected_url', + 'detected_match', + 'detected_lines', + 'detected_oss_lines', + 'detected_path', + ] file = sys.stdout if not output_file and self.output_file: output_file = self.output_file if output_file: file = open(output_file, 'w') writer = csv.DictWriter(file, fieldnames=fields) - writer.writeheader() # writing headers (field names) + writer.writeheader() # writing headers (field names) writer.writerows(csv_data) # writing data rows if output_file: file.close() @@ -206,6 +231,8 @@ def produce_from_str(self, json_str: str, output_file: str = None) -> bool: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return False return self.produce_from_json(data, output_file) + + # # End of CsvOutput Class # diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 92f87821..4e4ebba2 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os.path import sys @@ -64,17 +65,17 @@ def parse(self, data: json): file_details = data.get(f) # print(f'File: {f}: {file_details}') for d in file_details: - id_details = d.get("id") + id_details = d.get('id') if not id_details or id_details == 'none': continue purl = None if id_details == 'dependency': - dependencies = d.get("dependencies") + dependencies = d.get('dependencies') if not dependencies: self.print_stderr(f'Warning: No Dependencies found for {f}: {file_details}') continue for deps in dependencies: - purl = deps.get("purl") + purl = deps.get('purl') if not purl: self.print_stderr(f'Warning: No PURL found for {f}: {deps}') continue @@ -89,7 +90,7 @@ def parse(self, data: json): if licenses: dc = [] for lic in licenses: - name = lic.get("name") + name = lic.get('name') if name not in dc: # Only save the license name once fdl.append({'id': name}) dc.append(name) @@ -108,30 +109,33 @@ def parse(self, data: json): self.print_stderr(f'Warning: No PURL found for {f}: {file_details}') continue fd = {} - vulnerabilities = d.get("vulnerabilities") + vulnerabilities = d.get('vulnerabilities') if vulnerabilities: for vuln in vulnerabilities: - vuln_id = vuln.get("ID") + vuln_id = vuln.get('ID') if vuln_id == '': - vuln_id = vuln.get("id") + vuln_id = vuln.get('id') if not vuln_id or vuln_id == '': # Skip empty ids continue - vuln_cve = vuln.get("CVE", '') + vuln_cve = vuln.get('CVE', '') if vuln_cve == '': - vuln_cve = vuln.get("cve", '') - if vuln_id.upper().startswith("CPE:"): - fd['cpe'] = vuln_id # Save the component CPE if we have one + vuln_cve = vuln.get('cve', '') + if vuln_id.upper().startswith('CPE:'): + fd['cpe'] = vuln_id # Save the component CPE if we have one if vuln_cve != '': vuln_id = vuln_cve vd = vdx.get(vuln_id) # Check if we've already encountered this vulnerability if not vd: vuln_source = vuln.get('source', '').lower() - vd = {'cve': vuln_cve, - 'source': 'NVD' if vuln_source == 'nvd' else 'GitHub Advisories', - 'url': f'https://nvd.nist.gov/vuln/detail/{vuln_cve}' if vuln_source == 'nvd' else f'https://github.com/advisories/{vuln_id}', - 'severity': self._sev_lookup(vuln.get('severity', 'unknown').lower()), - 'affects': set() - } + vd = { + 'cve': vuln_cve, + 'source': 'NVD' if vuln_source == 'nvd' else 'GitHub Advisories', + 'url': f'https://nvd.nist.gov/vuln/detail/{vuln_cve}' + if vuln_source == 'nvd' + else f'https://github.com/advisories/{vuln_id}', + 'severity': self._sev_lookup(vuln.get('severity', 'unknown').lower()), + 'affects': set(), + } vd.get('affects').add(purl) vdx[vuln_id] = vd if cdx.get(purl): @@ -143,7 +147,7 @@ def parse(self, data: json): fdl = [] if licenses: for lic in licenses: - fdl.append({'id': lic.get("name")}) + fdl.append({'id': lic.get('name')}) fd['licenses'] = fdl cdx[purl] = fd # self.print_stderr(f'VD: {vdx}') @@ -190,7 +194,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'serialNumber': f'urn:uuid:{uuid.uuid4()}', 'version': 1, 'metadata': { - 'timestamp': datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + 'timestamp': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), 'tools': [ { 'vendor': 'SCANOSS', @@ -198,14 +202,10 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'version': __version__, } ], - 'component': { - 'type': 'application', - 'name': 'NOASSERTION', - 'version': 'NOASSERTION' - } + 'component': {'type': 'application', 'name': 'NOASSERTION', 'version': 'NOASSERTION'}, }, 'components': [], - 'vulnerabilities': [] + 'vulnerabilities': [], } for purl in cdx: comp = cdx.get(purl) @@ -230,7 +230,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'version': comp.get('version'), 'purl': purl, 'bom-ref': purl, - 'licenses': lic_text + 'licenses': lic_text, } cpe = comp.get('cpe', '') if cpe and cpe != '': @@ -250,7 +250,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'id': vuln_id, 'source': {'name': v_source, 'url': vulns.get('url')}, 'ratings': [{'severity': vulns.get('severity', 'unknown')}], - 'affects': affects + 'affects': affects, } data['vulnerabilities'].append(vd) # End for loop @@ -298,8 +298,10 @@ def _sev_lookup(value: str): 'low': 'low', 'info': 'info', 'none': 'none', - 'unknown': 'unknown' - }.get(value, 'unknown') + 'unknown': 'unknown', + }.get(value, 'unknown') + + # # End of CycloneDX Class # diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index 0182f948..85ee7ed7 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -61,9 +61,7 @@ '__pypackages__', } # Folder endings to skip -DEFAULT_SKIPPED_DIR_EXT = { - '.egg-info' -} +DEFAULT_SKIPPED_DIR_EXT = {'.egg-info'} # File extensions to skip DEFAULT_SKIPPED_EXT = { '.1', @@ -236,18 +234,18 @@ class FileFilters(ScanossBase): """ def __init__( - self, - debug: bool = False, - trace: bool = False, - quiet: bool = False, - scanoss_settings: 'ScanossSettings | None' = None, - all_extensions: bool = False, - all_folders: bool = False, - hidden_files_folders: bool = False, - operation_type: str = 'scanning', - skip_size: int = 0, - skip_extensions = None, - skip_folders = None + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + scanoss_settings: 'ScanossSettings | None' = None, + all_extensions: bool = False, + all_folders: bool = False, + hidden_files_folders: bool = False, + operation_type: str = 'scanning', + skip_size: int = 0, + skip_extensions=None, + skip_folders=None, ): """ Initialize scan filters based on default settings. Optionally append custom settings. diff --git a/src/scanoss/filecount.py b/src/scanoss/filecount.py index e3e9d9cc..a2f43b1b 100644 --- a/src/scanoss/filecount.py +++ b/src/scanoss/filecount.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2022, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import csv import os import pathlib @@ -36,9 +37,15 @@ class FileCount(ScanossBase): SCANOSS File Type Count class Handle the scanning of files, snippets and dependencies """ - def __init__(self, scan_output: str = None, hidden_files_folders: bool = False, - debug: bool = False, trace: bool = False, quiet: bool = False - ): + + def __init__( + self, + scan_output: str = None, + hidden_files_folders: bool = False, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + ): """ Initialise scanning class """ @@ -56,7 +63,7 @@ def __filter_files(self, files: list) -> list: file_list = [] for f in files: ignore = False - if f.startswith(".") and not self.hidden_files_folders: # Ignore all . files unless requested + if f.startswith('.') and not self.hidden_files_folders: # Ignore all . files unless requested ignore = True if not ignore: file_list.append(f) @@ -71,7 +78,7 @@ def __filter_dirs(self, dirs: list) -> list: dir_list = [] for d in dirs: ignore = False - if d.startswith(".") and not self.hidden_files_folders: # Ignore all . folders unless requested + if d.startswith('.') and not self.hidden_files_folders: # Ignore all . folders unless requested ignore = True if not ignore: dir_list.append(d) @@ -84,7 +91,7 @@ def __log_result(self, string, outfile=None): if not outfile and self.scan_output: outfile = self.scan_output if outfile: - with open(outfile, "a") as rf: + with open(outfile, 'a') as rf: rf.write(string + '\n') else: print(string) @@ -98,9 +105,9 @@ def count_files(self, scan_dir: str) -> bool: """ success = True if not scan_dir: - raise Exception(f"ERROR: Please specify a folder to scan") + raise Exception(f'ERROR: Please specify a folder to scan') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): - raise Exception(f"ERROR: Specified folder does not exist or is not a folder: {scan_dir}") + raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') self.print_msg(f'Searching {scan_dir} for files to count...') spinner = None @@ -111,17 +118,17 @@ def count_files(self, scan_dir: str) -> bool: file_size = 0 for root, dirs, files in os.walk(scan_dir): self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') - dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories - filtered_files = self.__filter_files(files) # Strip out unwanted files + dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories + filtered_files = self.__filter_files(files) # Strip out unwanted files self.print_trace(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}') - for file in filtered_files: # Cycle through each filtered file + for file in filtered_files: # Cycle through each filtered file path = os.path.join(root, file) f_size = 0 try: f_size = os.stat(path).st_size except Exception as e: self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # broken symlink - if f_size > 0: # Ignore broken links and empty files + if f_size > 0: # Ignore broken links and empty files file_count = file_count + 1 file_size = file_size + f_size f_suffix = pathlib.Path(file).suffix @@ -140,18 +147,18 @@ def count_files(self, scan_dir: str) -> bool: # End for loop if spinner: spinner.finish() - self.print_stderr(f'Found {file_count:,.0f} files with a total size of {file_size/(1<<20):,.2f} MB.') + self.print_stderr(f'Found {file_count:,.0f} files with a total size of {file_size / (1 << 20):,.2f} MB.') if file_types: csv_dict = [] for k in file_types: d = file_types[k] - csv_dict.append({'extension': k, 'count': d[0], 'size(MB)': f'{d[1]/(1<<20):,.2f}'}) + csv_dict.append({'extension': k, 'count': d[0], 'size(MB)': f'{d[1] / (1 << 20):,.2f}'}) fields = ['extension', 'count', 'size(MB)'] file = sys.stdout if self.scan_output: file = open(self.scan_output, 'w') writer = csv.DictWriter(file, fieldnames=fields) - writer.writeheader() # writing headers (field names) + writer.writeheader() # writing headers (field names) writer.writerows(csv_dict) # writing data rows if self.scan_output: file.close() diff --git a/src/scanoss/inspection/__init__.py b/src/scanoss/inspection/__init__.py index 5243c416..ebd5917f 100644 --- a/src/scanoss/inspection/__init__.py +++ b/src/scanoss/inspection/__init__.py @@ -1,23 +1,23 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2024, SCANOSS + Copyright (c) 2024, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index 40da6b2d..ebee8211 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -1,52 +1,64 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json from typing import Dict, Any from .policy_check import PolicyCheck, PolicyStatus + class Copyleft(PolicyCheck): """ SCANOSS Copyleft class Inspects components for copyleft licenses """ - def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, - format_type: str = 'json', status: str = None, output: str = None, include: str = None, - exclude: str = None, explicit: str = None): + def __init__( + self, + debug: bool = False, + trace: bool = True, + quiet: bool = False, + filepath: str = None, + format_type: str = 'json', + status: str = None, + output: str = None, + include: str = None, + exclude: str = None, + explicit: str = None, + ): """ - Initialize the Copyleft class. - - :param debug: Enable debug mode - :param trace: Enable trace mode (default True) - :param quiet: Enable quiet mode - :param filepath: Path to the file containing component data - :param format_type: Output format ('json' or 'md') - :param status: Path to save the status output - :param output: Path to save detailed output - :param include: Licenses to include in the analysis - :param exclude: Licenses to exclude from the analysis - :param explicit: Explicitly defined licenses + Initialize the Copyleft class. + + :param debug: Enable debug mode + :param trace: Enable trace mode (default True) + :param quiet: Enable quiet mode + :param filepath: Path to the file containing component data + :param format_type: Output format ('json' or 'md') + :param status: Path to save the status output + :param output: Path to save detailed output + :param include: Licenses to include in the analysis + :param exclude: Licenses to exclude from the analysis + :param explicit: Explicitly defined licenses """ super().__init__(debug, trace, quiet, filepath, format_type, status, output, name='Copyleft Policy') self.license_util.init(include, exclude, explicit) @@ -58,23 +70,22 @@ def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, self.exclude = exclude self.explicit = explicit - def _json(self, components: list) -> Dict[str, Any]: """ - Format the components with copyleft licenses as JSON. + Format the components with copyleft licenses as JSON. - :param components: List of components with copyleft licenses - :return: Dictionary with formatted JSON details and summary + :param components: List of components with copyleft licenses + :return: Dictionary with formatted JSON details and summary """ details = {} if len(components) > 0: - details = { 'components': components } + details = {'components': components} return { - 'details': f'{json.dumps(details, indent=2)}\n', - 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n' + 'details': f'{json.dumps(details, indent=2)}\n', + 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n', } - def _markdown(self, components: list) -> Dict[str,Any]: + def _markdown(self, components: list) -> Dict[str, Any]: """ Format the components with copyleft licenses as Markdown. @@ -83,7 +94,7 @@ def _markdown(self, components: list) -> Dict[str,Any]: """ headers = ['Component', 'Version', 'License', 'URL', 'Copyleft'] centered_columns = [1, 4] - rows: [[]]= [] + rows: [[]] = [] for component in components: for lic in component['licenses']: row = [ @@ -91,17 +102,17 @@ def _markdown(self, components: list) -> Dict[str,Any]: component['version'], lic['spdxid'], lic['url'], - 'YES' if lic['copyleft'] else 'NO' + 'YES' if lic['copyleft'] else 'NO', ] rows.append(row) # End license loop # End component loop - return { - 'details': f'### Copyleft licenses\n{self.generate_table(headers,rows,centered_columns)}\n', - 'summary' : f'{len(components)} component(s) with copyleft licenses were found.\n' + return { + 'details': f'### Copyleft licenses\n{self.generate_table(headers, rows, centered_columns)}\n', + 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n', } - def _jira_markdown(self, components: list) -> Dict[str,Any]: + def _jira_markdown(self, components: list) -> Dict[str, Any]: """ Format the components with copyleft licenses as Markdown. @@ -110,7 +121,7 @@ def _jira_markdown(self, components: list) -> Dict[str,Any]: """ headers = ['Component', 'Version', 'License', 'URL', 'Copyleft'] centered_columns = [1, 4] - rows: [[]]= [] + rows: [[]] = [] for component in components: for lic in component['licenses']: row = [ @@ -118,22 +129,22 @@ def _jira_markdown(self, components: list) -> Dict[str,Any]: component['version'], lic['spdxid'], lic['url'], - 'YES' if lic['copyleft'] else 'NO' + 'YES' if lic['copyleft'] else 'NO', ] rows.append(row) # End license loop # End component loop - return { - 'details': f'{self.generate_jira_table(headers,rows,centered_columns)}', - 'summary' : f'{len(components)} component(s) with copyleft licenses were found.\n' + return { + 'details': f'{self.generate_jira_table(headers, rows, centered_columns)}', + 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n', } def _filter_components_with_copyleft_licenses(self, components: list) -> list: """ - Filter the components list to include only those with copyleft licenses. + Filter the components list to include only those with copyleft licenses. - :param components: List of all components - :return: List of components with copyleft licenses + :param components: List of all components + :return: List of components with copyleft licenses """ filtered_components = [] for component in components: @@ -179,6 +190,8 @@ def run(self): if len(copyleft_components) <= 0: return PolicyStatus.FAIL.value, results return PolicyStatus.SUCCESS.value, results + + # # End of Copyleft Class # diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index 8f392660..a7e352c8 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os.path from abc import abstractmethod @@ -39,13 +40,17 @@ class PolicyStatus(Enum): FAIL (int): Indicates that the policy check failed (value: 1). ERROR (int): Indicates that an error occurred during the policy check (value: 2). """ + SUCCESS = 0 FAIL = 1 ERROR = 2 + + # # End of PolicyStatus Class # + class ComponentID(Enum): """ Enumeration representing different types of software components. @@ -55,13 +60,17 @@ class ComponentID(Enum): SNIPPET (str): Represents a code snippet component (value: "snippet"). DEPENDENCY (str): Represents a dependency component (value: "dependency"). """ - FILE = "file" - SNIPPET = "snippet" - DEPENDENCY = "dependency" + + FILE = 'file' + SNIPPET = 'snippet' + DEPENDENCY = 'dependency' + + # # End of ComponentID Class # + class PolicyCheck(ScanossBase): """ A base class for implementing various software policy checks. @@ -78,8 +87,17 @@ class PolicyCheck(ScanossBase): VALID_FORMATS = {'md', 'json', 'jira_md'} - def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, - format_type: str = None, status: str = None, output: str = None, name: str = None): + def __init__( + self, + debug: bool = False, + trace: bool = True, + quiet: bool = False, + filepath: str = None, + format_type: str = None, + status: str = None, + output: str = None, + name: str = None, + ): super().__init__(debug, trace, quiet) self.license_util = LicenseUtil() self.filepath = filepath @@ -110,14 +128,14 @@ def run(self): @abstractmethod def _json(self, components: list) -> Dict[str, Any]: """ - Format the policy checks results as JSON. - This method should be implemented by subclasses to create a Markdown representation - of the policy check results. - - :param components: List of components to be formatted. - :return: A dictionary containing two keys: - - 'details': A JSON-formatted string with the full list of components - - 'summary': A string summarizing the number of components found + Format the policy checks results as JSON. + This method should be implemented by subclasses to create a Markdown representation + of the policy check results. + + :param components: List of components to be formatted. + :return: A dictionary containing two keys: + - 'details': A JSON-formatted string with the full list of components + - 'summary': A string summarizing the number of components found """ pass @@ -147,8 +165,9 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: """ pass - def _append_component(self,components: Dict[str, Any], new_component: Dict[str, Any], - id: str, status: str) -> Dict[str, Any]: + def _append_component( + self, components: Dict[str, Any], new_component: Dict[str, Any], id: str, status: str + ) -> Dict[str, Any]: """ Append a new component to the component's dictionary. @@ -169,12 +188,12 @@ def _append_component(self,components: Dict[str, Any], new_component: Dict[str, else: purl = new_component['purl'] - component_key = f"{purl}@{new_component['version']}" + component_key = f'{purl}@{new_component["version"]}' components[component_key] = { - 'purl': purl, - 'version': new_component['version'], - 'licenses': {}, - 'status': status, + 'purl': purl, + 'version': new_component['version'], + 'licenses': {}, + 'status': status, } if not new_component.get('licenses'): @@ -191,16 +210,16 @@ def _append_component(self,components: Dict[str, Any], new_component: Dict[str, } return components - def _get_components_from_results(self,results: Dict[str, Any]) -> list or None: + def _get_components_from_results(self, results: Dict[str, Any]) -> list or None: """ - Process the results dictionary to extract and format component information. + Process the results dictionary to extract and format component information. - This function iterates through the results dictionary, identifying components from - different sources (files, snippets, and dependencies). It consolidates this information - into a list of unique components, each with its associated licenses and other details. + This function iterates through the results dictionary, identifying components from + different sources (files, snippets, and dependencies). It consolidates this information + into a list of unique components, each with its associated licenses and other details. - :param results: A dictionary containing the raw results of a component scan - :return: A list of dictionaries, each representing a unique component with its details + :param results: A dictionary containing the raw results of a component scan + :return: A list of dictionaries, each representing a unique component with its details """ if results is None: self.print_stderr(f'ERROR: Results cannot be empty') @@ -226,7 +245,7 @@ def _get_components_from_results(self,results: Dict[str, Any]) -> list or None: if not c.get('version'): self.print_stderr(f'WARNING: Result missing version. Skipping.') continue - component_key = f"{c['purl'][0]}@{c['version']}" + component_key = f'{c["purl"][0]}@{c["version"]}' # Initialize or update the component entry if component_key not in components: components = self._append_component(components, c, component_id, status) @@ -244,7 +263,7 @@ def _get_components_from_results(self,results: Dict[str, Any]) -> list or None: if not d.get('version'): self.print_stderr(f'WARNING: Result missing version. Skipping.') continue - component_key = f"{d['purl']}@{d['version']}" + component_key = f'{d["purl"]}@{d["version"]}' if component_key not in components: components = self._append_component(components, d, component_id, status) # End of dependencies loop @@ -271,11 +290,13 @@ def generate_table(self, headers, rows, centered_columns=None): if headers is None: self.print_stderr('ERROR: Header are no set') return None + # Decide which separator to use def create_separator(index): if centered_columns is None: return '-' return ':-:' if index in centered_column_set else '-' + # Build the row separator row_separator = col_sep + col_sep.join(create_separator(index) for index, _ in enumerate(headers)) + col_sep # build table rows @@ -297,7 +318,7 @@ def generate_jira_table(self, headers, rows, centered_columns=None): return table - def _get_formatter(self)-> Callable[[List[dict]], Dict[str,Any]] or None: + def _get_formatter(self) -> Callable[[List[dict]], Dict[str, Any]] or None: """ Get the appropriate formatter function based on the specified format. @@ -348,11 +369,11 @@ def _load_input_file(self): Returns: Dict[str, Any]: The parsed JSON data - """ + """ if not os.path.exists(self.filepath): self.print_stderr(f'ERROR: The file "{self.filepath}" does not exist.') return None - with open(self.filepath, "r") as jsonfile: + with open(self.filepath, 'r') as jsonfile: try: return json.load(jsonfile) except Exception as e: @@ -381,6 +402,8 @@ def _get_components(self): return None components = self._get_components_from_results(self.results) return components + + # # End of PolicyCheck Class # diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index 5d43ff88..5b222406 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -1,38 +1,49 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json from typing import Dict, Any from .policy_check import PolicyCheck, PolicyStatus + class UndeclaredComponent(PolicyCheck): """ SCANOSS UndeclaredComponent class Inspects for undeclared components """ - def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, filepath: str = None, - format_type: str = 'json', status: str = None, output: str = None, sbom_format: str = 'settings'): + def __init__( + self, + debug: bool = False, + trace: bool = True, + quiet: bool = False, + filepath: str = None, + format_type: str = 'json', + status: str = None, + output: str = None, + sbom_format: str = 'settings', + ): """ Initialize the UndeclaredComponent class. @@ -45,20 +56,21 @@ def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False, :param output: Path to save detailed output :param sbom_format: Sbom format for status output (default 'settings') """ - super().__init__(debug, trace, quiet, filepath, format_type, status, output, - name='Undeclared Components Policy') + super().__init__( + debug, trace, quiet, filepath, format_type, status, output, name='Undeclared Components Policy' + ) self.filepath = filepath self.format = format self.output = output self.status = status self.sbom_format = sbom_format - def _get_undeclared_component(self, components: list)-> list or None: + def _get_undeclared_component(self, components: list) -> list or None: """ - Filter the components list to include only undeclared components. + Filter the components list to include only undeclared components. - :param components: List of all components - :return: List of undeclared components + :param components: List of all components + :return: List of undeclared components """ if components is None: self.print_debug(f'WARNING: No components provided!') @@ -73,24 +85,29 @@ def _get_undeclared_component(self, components: list)-> list or None: def _get_jira_summary(self, components: list) -> str: """ - Get a summary of the undeclared components. + Get a summary of the undeclared components. - :param components: List of all components - :return: Component summary markdown - """ + :param components: List of all components + :return: Component summary markdown + """ if len(components) > 0: if self.sbom_format == 'settings': - json_str = (json.dumps(self._generate_scanoss_file(components), indent=2).replace('\n', '\\n') - .replace('"', '\\"')) + json_str = ( + json.dumps(self._generate_scanoss_file(components), indent=2) + .replace('\n', '\\n') + .replace('"', '\\"') + ) return f'{len(components)} undeclared component(s) were found.\nAdd the following snippet into your `scanoss.json` file\n{{code:json}}\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n{{code}}\n' else: - json_str = (json.dumps(self._generate_scanoss_file(components), indent=2).replace('\n', '\\n') - .replace('"', '\\"')) + json_str = ( + json.dumps(self._generate_scanoss_file(components), indent=2) + .replace('\n', '\\n') + .replace('"', '\\"') + ) return f'{len(components)} undeclared component(s) were found.\nAdd the following snippet into your `sbom.json` file\n{{code:json}}\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n{{code}}\n' return f'{len(components)} undeclared component(s) were found.\\n' - def _get_summary(self, components: list) -> str: """ Get a summary of the undeclared components. @@ -101,11 +118,15 @@ def _get_summary(self, components: list) -> str: summary = f'{len(components)} undeclared component(s) were found.\n' if len(components) > 0: if self.sbom_format == 'settings': - summary += (f'Add the following snippet into your `scanoss.json` file\n' - f'\n```json\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n```\n') + summary += ( + f'Add the following snippet into your `scanoss.json` file\n' + f'\n```json\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n```\n' + ) else: - summary += (f'Add the following snippet into your `sbom.json` file\n' - f'\n```json\n{json.dumps(self._generate_sbom_file(components), indent=2)}\n```\n') + summary += ( + f'Add the following snippet into your `sbom.json` file\n' + f'\n```json\n{json.dumps(self._generate_sbom_file(components), indent=2)}\n```\n' + ) return summary @@ -120,71 +141,71 @@ def _json(self, components: list) -> Dict[str, Any]: if len(components) > 0: details = {'components': components} return { - 'details': f'{json.dumps(details, indent=2)}\n', + 'details': f'{json.dumps(details, indent=2)}\n', 'summary': self._get_summary(components), } - def _markdown(self, components: list) -> Dict[str,Any]: + def _markdown(self, components: list) -> Dict[str, Any]: """ - Format the undeclared components as Markdown. + Format the undeclared components as Markdown. - :param components: List of undeclared components - :return: Dictionary with formatted Markdown details and summary - """ + :param components: List of undeclared components + :return: Dictionary with formatted Markdown details and summary + """ headers = ['Component', 'Version', 'License'] - rows: [[]]= [] + rows: [[]] = [] # TODO look at using SpdxLite license name lookup method for component in components: - licenses = " - ".join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) + licenses = ' - '.join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) rows.append([component['purl'], component['version'], licenses]) - return { - 'details': f'### Undeclared components\n{self.generate_table(headers,rows)}\n', + return { + 'details': f'### Undeclared components\n{self.generate_table(headers, rows)}\n', 'summary': self._get_summary(components), } - def _jira_markdown(self, components: list) -> Dict[str,Any]: + def _jira_markdown(self, components: list) -> Dict[str, Any]: """ - Format the undeclared components as Markdown. + Format the undeclared components as Markdown. - :param components: List of undeclared components - :return: Dictionary with formatted Markdown details and summary - """ + :param components: List of undeclared components + :return: Dictionary with formatted Markdown details and summary + """ headers = ['Component', 'Version', 'License'] - rows: [[]]= [] + rows: [[]] = [] # TODO look at using SpdxLite license name lookup method for component in components: - licenses = " - ".join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) + licenses = ' - '.join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) rows.append([component['purl'], component['version'], licenses]) - return { - 'details': f'{self.generate_jira_table(headers,rows)}', + return { + 'details': f'{self.generate_jira_table(headers, rows)}', 'summary': self._get_jira_summary(components), } def _get_unique_components(self, components: list) -> list: """ - Generate a list of unique components. + Generate a list of unique components. - :param components: List of undeclared components - :return: list of unique components - """ + :param components: List of undeclared components + :return: list of unique components + """ unique_components = {} if components is None: self.print_stderr(f'WARNING: No components provided!') return [] for component in components: - unique_components[component['purl']] = {'purl': component['purl']} + unique_components[component['purl']] = {'purl': component['purl']} return list(unique_components.values()) def _generate_scanoss_file(self, components: list) -> dict: """ - Generate a list of PURLs for the scanoss.json file. + Generate a list of PURLs for the scanoss.json file. - :param components: List of undeclared components - :return: scanoss.json Dictionary - """ + :param components: List of undeclared components + :return: scanoss.json Dictionary + """ scanoss_settings = { - 'bom':{ + 'bom': { 'include': self._get_unique_components(components), } } @@ -193,11 +214,11 @@ def _generate_scanoss_file(self, components: list) -> dict: def _generate_sbom_file(self, components: list) -> dict: """ - Generate a list of PURLs for the SBOM file. + Generate a list of PURLs for the SBOM file. - :param components: List of undeclared components - :return: SBOM Dictionary with components - """ + :param components: List of undeclared components + :return: SBOM Dictionary with components + """ sbom = { 'components': self._get_unique_components(components), } @@ -236,6 +257,8 @@ def run(self): if len(undeclared_components) <= 0: return PolicyStatus.FAIL.value, results return PolicyStatus.SUCCESS.value, results + + # # End of UndeclaredComponent Class # diff --git a/src/scanoss/inspection/utils/license_utils.py b/src/scanoss/inspection/utils/license_utils.py index f97b2758..beb7dd09 100644 --- a/src/scanoss/inspection/utils/license_utils.py +++ b/src/scanoss/inspection/utils/license_utils.py @@ -1,61 +1,82 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + from ...scanossbase import ScanossBase DEFAULT_COPYLEFT_LICENSES = { - 'agpl-3.0-only', 'artistic-1.0', 'artistic-2.0', 'cc-by-sa-4.0', 'cddl-1.0', 'cddl-1.1', 'cecill-2.1', - 'epl-1.0', 'epl-2.0', 'gfdl-1.1-only', 'gfdl-1.2-only', 'gfdl-1.3-only', 'gpl-1.0-only', 'gpl-2.0-only', - 'gpl-3.0-only', 'lgpl-2.1-only', 'lgpl-3.0-only', 'mpl-1.1', 'mpl-2.0', 'sleepycat', 'watcom-1.0' + 'agpl-3.0-only', + 'artistic-1.0', + 'artistic-2.0', + 'cc-by-sa-4.0', + 'cddl-1.0', + 'cddl-1.1', + 'cecill-2.1', + 'epl-1.0', + 'epl-2.0', + 'gfdl-1.1-only', + 'gfdl-1.2-only', + 'gfdl-1.3-only', + 'gpl-1.0-only', + 'gpl-2.0-only', + 'gpl-3.0-only', + 'lgpl-2.1-only', + 'lgpl-3.0-only', + 'mpl-1.1', + 'mpl-2.0', + 'sleepycat', + 'watcom-1.0', } + class LicenseUtil(ScanossBase): """ - A utility class for handling software licenses, particularly copyleft licenses. + A utility class for handling software licenses, particularly copyleft licenses. - This class provides functionality to initialize, manage, and query a set of - copyleft licenses. It also offers a method to generate URLs for license information. + This class provides functionality to initialize, manage, and query a set of + copyleft licenses. It also offers a method to generate URLs for license information. """ + BASE_SPDX_ORG_URL = 'https://spdx.org/licenses' BASE_OSADL_URL = 'https://www.osadl.org/fileadmin/checklists/unreflicenses' - def __init__(self,debug: bool = False, trace: bool = True, quiet: bool = False): + def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False): super().__init__(debug, trace, quiet) self.default_copyleft_licenses = set(DEFAULT_COPYLEFT_LICENSES) self.copyleft_licenses = set() def init(self, include: str = None, exclude: str = None, explicit: str = None): """ - Initialize the set of copyleft licenses based on user input. + Initialize the set of copyleft licenses based on user input. - This method allows for customization of the copyleft license set by: - - Setting an explicit list of licenses - - Including additional licenses to the default set - - Excluding specific licenses from the default set + This method allows for customization of the copyleft license set by: + - Setting an explicit list of licenses + - Including additional licenses to the default set + - Excluding specific licenses from the default set - :param include: Comma-separated string of licenses to include - :param exclude: Comma-separated string of licenses to exclude - :param explicit: Comma-separated string of licenses to use exclusively + :param include: Comma-separated string of licenses to include + :param exclude: Comma-separated string of licenses to exclude + :param explicit: Comma-separated string of licenses to use exclusively """ if self.debug: self.print_stderr(f'Include Copyleft licenses: ${include}') @@ -73,7 +94,7 @@ def init(self, include: str = None, exclude: str = None, explicit: str = None): if include: include = include.strip() if include: - inc =[item.strip().lower() for item in include.split(',')] + inc = [item.strip().lower() for item in include.split(',')] self.copyleft_licenses.update(inc) if exclude: exclude = exclude.strip() @@ -85,23 +106,22 @@ def init(self, include: str = None, exclude: str = None, explicit: str = None): def is_copyleft(self, spdxid: str) -> bool: """ - Check if a given license is considered copyleft. + Check if a given license is considered copyleft. - :param spdxid: The SPDX identifier of the license to check - :return: True if the license is copyleft, False otherwise + :param spdxid: The SPDX identifier of the license to check + :return: True if the license is copyleft, False otherwise """ return spdxid.lower() in self.copyleft_licenses def get_spdx_url(self, spdxid: str) -> str: """ - Generate the URL for the SPDX page of a license. + Generate the URL for the SPDX page of a license. - :param spdxid: The SPDX identifier of the license - :return: The URL of the SPDX page for the given license + :param spdxid: The SPDX identifier of the license + :return: The URL of the SPDX page for the given license """ return f'{self.BASE_SPDX_ORG_URL}/{spdxid}.html' - def get_osadl_url(self, spdxid: str) -> str: """ Generate the URL for the OSADL (Open Source Automation Development Lab) page of a license. @@ -110,6 +130,8 @@ def get_osadl_url(self, spdxid: str) -> str: :return: The URL of the OSADL page for the given license """ return f'{self.BASE_OSADL_URL}/{spdxid}.txt' + + # # End of LicenseUtil Class # diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 438386d1..8ca80aa3 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -1,25 +1,25 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ import json @@ -27,27 +27,27 @@ from .scanossbase import ScanossBase -MATCH_TYPES = ["file", "snippet"] -STATUSES = ["pending", "identified"] +MATCH_TYPES = ['file', 'snippet'] +STATUSES = ['pending', 'identified'] AVAILABLE_FILTER_VALUES = { - "match_type": [e for e in MATCH_TYPES], - "status": [e for e in STATUSES], + 'match_type': [e for e in MATCH_TYPES], + 'status': [e for e in STATUSES], } ARG_TO_FILTER_MAP = { - "match_type": "id", - "status": "status", + 'match_type': 'id', + 'status': 'status', } PENDING_IDENTIFICATION_FILTERS = { - "match_type": ["file", "snippet"], - "status": ["pending"], + 'match_type': ['file', 'snippet'], + 'status': ['pending'], } -AVAILABLE_OUTPUT_FORMATS = ["json", "plain"] +AVAILABLE_OUTPUT_FORMATS = ['json', 'plain'] class Results(ScanossBase): @@ -95,11 +95,11 @@ def load_file(self, file: str) -> Dict[str, Any]: Returns: Dict[str, Any]: The parsed JSON data """ - with open(file, "r") as jsonfile: + with open(file, 'r') as jsonfile: try: return json.load(jsonfile) except Exception as e: - self.print_stderr(f"ERROR: Problem parsing input JSON: {e}") + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: """ @@ -143,7 +143,7 @@ def _load_filters(self, **kwargs): @staticmethod def _extract_comma_separated_values(values: str): - return [value.strip() for value in values.split(",")] + return [value.strip() for value in values.split(',')] def apply_filters(self): """Apply the filters to the data""" @@ -172,14 +172,11 @@ def _item_matches_filters(self, item): @staticmethod def _validate_filter_values(filter_key: str, filter_value: List[str]): - if any( - value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) - for value in filter_value - ): - valid_values = ", ".join(AVAILABLE_FILTER_VALUES.get(filter_key, [])) + if any(value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) for value in filter_value): + valid_values = ', '.join(AVAILABLE_FILTER_VALUES.get(filter_key, [])) raise Exception( f"ERROR: Invalid filter value '{filter_value}' for filter '{filter_key.value}'. " - f"Valid values are: {valid_values}" + f'Valid values are: {valid_values}' ) def get_pending_identifications(self): @@ -226,9 +223,7 @@ def _present_json(self, file: str = None): Args: file (str, optional): Output file. Defaults to None. """ - self.print_to_file_or_stdout( - json.dumps(self._format_json_output(), indent=2), file - ) + self.print_to_file_or_stdout(json.dumps(self._format_json_output(), indent=2), file) def _format_json_output(self): """ @@ -240,15 +235,11 @@ def _format_json_output(self): formatted_data.append( { 'file': item.get('filename'), - 'status': item.get('status', "N/A"), + 'status': item.get('status', 'N/A'), 'match_type': item['id'], - 'matched': item.get('matched', "N/A"), - 'purl': (item.get('purl')[0] if item.get('purl') else "N/A"), - 'license': ( - item.get('licenses')[0].get('name', "N/A") - if item.get('licenses') - else "N/A" - ), + 'matched': item.get('matched', 'N/A'), + 'purl': (item.get('purl')[0] if item.get('purl') else 'N/A'), + 'license': (item.get('licenses')[0].get('name', 'N/A') if item.get('licenses') else 'N/A'), } ) return {'results': formatted_data, 'total': len(formatted_data)} @@ -263,7 +254,7 @@ def _present_plain(self, file: str = None): None """ if not self.data: - return self.print_stderr("No results to present") + return self.print_stderr('No results to present') self.print_to_file_or_stdout(self._format_plain_output(), file) def _present_stdout(self): @@ -273,7 +264,7 @@ def _present_stdout(self): None """ if not self.data: - return self.print_stderr("No results to present") + return self.print_stderr('No results to present') self.print_to_file_or_stdout(self._format_plain_output()) def _format_plain_output(self): @@ -281,9 +272,9 @@ def _format_plain_output(self): Format the output data into a plain text string """ - formatted = "" + formatted = '' for item in self.data: - formatted += f"{self._format_plain_output_item(item)} \n" + formatted += f'{self._format_plain_output_item(item)} \n' return formatted @staticmethod @@ -292,10 +283,10 @@ def _format_plain_output_item(item): licenses = item.get('licenses', []) return ( - f"File: {item.get('filename')}\n" - f"Match type: {item.get('id')}\n" - f"Status: {item.get('status', 'N/A')}\n" - f"Matched: {item.get('matched', 'N/A')}\n" - f"Purl: {purls[0] if purls else 'N/A'}\n" - f"License: {licenses[0].get('name', 'N/A') if licenses else 'N/A'}\n" + f'File: {item.get("filename")}\n' + f'Match type: {item.get("id")}\n' + f'Status: {item.get("status", "N/A")}\n' + f'Matched: {item.get("matched", "N/A")}\n' + f'Purl: {purls[0] if purls else "N/A"}\n' + f'License: {licenses[0].get("name", "N/A") if licenses else "N/A"}\n' ) diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index 7527d0fd..677c7774 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -1,25 +1,25 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ import json @@ -33,8 +33,17 @@ class ScancodeDeps(ScanossBase): """ SCANOSS dependency scanning class """ - def __init__(self, debug: bool = False, quiet: bool = False, trace: bool = False, output_file: str = None, - scan_output: str = None, timeout: int = 600, sc_command: str = None): + + def __init__( + self, + debug: bool = False, + quiet: bool = False, + trace: bool = False, + output_file: str = None, + scan_output: str = None, + timeout: int = 600, + sc_command: str = None, + ): """ Initialise ScancodeDeps class """ @@ -54,12 +63,11 @@ def __log_result(self, string, outfile=None): if not outfile and self.scan_output: outfile = self.scan_output if outfile: - with open(outfile, "a") as rf: + with open(outfile, 'a') as rf: rf.write(string + '\n') else: print(string) - def remove_interim_file(self, output_file: str = None): """ Remove the temporary Scancode interim file @@ -86,7 +94,7 @@ def produce_from_json(self, data: json) -> dict: self.print_debug(f'Processing Scancode results into Dependency data...') files = [] for t in data: - if t == 'files': # Only interested in 'files' details + if t == 'files': # Only interested in 'files' details files_details = data.get(t) if not files_details or files_details == '': continue @@ -121,7 +129,7 @@ def produce_from_json(self, data: json) -> dict: dp_data = {'purl': dp} rq = d.get('extracted_requirement') # scancode format 2.0 if not rq or rq == '': - rq = d.get('requirement') # scancode format 1.0 + rq = d.get('requirement') # scancode format 1.0 # skip requirement if it ends with the purl (i.e. exact version) or if it's local (file) if rq and rq != '' and not dp.endswith(rq) and not rq.startswith('file:'): dp_data['requirement'] = rq @@ -206,17 +214,32 @@ def run_scan(self, output_file: str = None, what_to_scan: str = None) -> bool: output_file = self.output_file try: open(output_file, 'w').close() - self.print_trace(f'About to execute {self.sc_command} -p --only-findings --quiet --json {output_file}' - f' {what_to_scan}') - result = subprocess.run([self.sc_command, '-p', '--only-findings', '--quiet', '--strip-root', '--json', - output_file, what_to_scan], - cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, timeout=self.timeout - ) + self.print_trace( + f'About to execute {self.sc_command} -p --only-findings --quiet --json {output_file} {what_to_scan}' + ) + result = subprocess.run( + [ + self.sc_command, + '-p', + '--only-findings', + '--quiet', + '--strip-root', + '--json', + output_file, + what_to_scan, + ], + cwd=os.getcwd(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=self.timeout, + ) self.print_trace(f'Subprocess return: {result}') if result.returncode: - self.print_stderr(f'ERROR: Scancode dependency scan of {what_to_scan} failed with exit code' - f' {result.returncode}:\n{result.stdout}') + self.print_stderr( + f'ERROR: Scancode dependency scan of {what_to_scan} failed with exit code' + f' {result.returncode}:\n{result.stdout}' + ) return False except subprocess.TimeoutExpired as e: self.print_stderr(f'ERROR: Timed out attempting to run scancode dependency scan on {what_to_scan}: {e}') @@ -245,21 +268,21 @@ def load_from_file(self, json_file: str = None) -> json: self.print_stderr(f'ERROR: Problem loading input JSON: {e}') return None - @staticmethod - def __remove_dep_scope(deps: json)->json: + def __remove_dep_scope(deps: json) -> json: """ :param deps: dependencies with scopes :return dependencies without scopes """ - files = deps.get("files") + files = deps.get('files') for file in files: if 'purls' in file: - purls = file.get("purls") + purls = file.get('purls') for purl in purls: - purl.pop("scope",None) + purl.pop('scope', None) + + return {'files': files} - return {"files": files } # # End of ScancodeDeps Class diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index e945742d..79f81d5b 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os from pathlib import Path @@ -52,12 +53,13 @@ FAST_WINNOWING = False try: from scanoss_winnowing.winnowing import Winnowing + FAST_WINNOWING = True except ModuleNotFoundError or ImportError: FAST_WINNOWING = False from .winnowing import Winnowing -WFP_FILE_START = "file=" +WFP_FILE_START = 'file=' MAX_POST_SIZE = 64 * 1024 # 64k Max post size @@ -103,7 +105,7 @@ def __init__( strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None, - scan_settings: 'ScanossSettings | None' = None + scan_settings: 'ScanossSettings | None' = None, ): """ Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning @@ -113,7 +115,7 @@ def __init__( skip_folders = [] if skip_extensions is None: skip_extensions = [] - self.wfp = wfp if wfp else "scanner_output.wfp" + self.wfp = wfp if wfp else 'scanner_output.wfp' self.scan_output = scan_output self.output_format = output_format self.no_wfp_file = no_wfp_file @@ -129,26 +131,51 @@ def __init__( self.skip_extensions = skip_extensions ver_details = Scanner.version_details() - self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=self._skip_snippets, - all_extensions=all_extensions, obfuscate=obfuscate, hpsm=self.hpsm, - strip_hpsm_ids=strip_hpsm_ids, strip_snippet_ids=strip_snippet_ids, - skip_md5_ids=skip_md5_ids - ) - self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url, - flags=flags, timeout=timeout, - ver_details=ver_details, ignore_cert_errors=ignore_cert_errors, - proxy=proxy, ca_cert=ca_cert, pac=pac, retry=retry - ) + self.winnowing = Winnowing( + debug=debug, + quiet=quiet, + skip_snippets=self._skip_snippets, + all_extensions=all_extensions, + obfuscate=obfuscate, + hpsm=self.hpsm, + strip_hpsm_ids=strip_hpsm_ids, + strip_snippet_ids=strip_snippet_ids, + skip_md5_ids=skip_md5_ids, + ) + self.scanoss_api = ScanossApi( + debug=debug, + trace=trace, + quiet=quiet, + api_key=api_key, + url=url, + flags=flags, + timeout=timeout, + ver_details=ver_details, + ignore_cert_errors=ignore_cert_errors, + proxy=proxy, + ca_cert=ca_cert, + pac=pac, + retry=retry, + ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) - grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key, - ver_details=ver_details, ca_cert=ca_cert, proxy=proxy, pac=pac, grpc_proxy=grpc_proxy - ) + grpc_api = ScanossGrpc( + url=grpc_url, + debug=debug, + quiet=quiet, + trace=trace, + api_key=api_key, + ver_details=ver_details, + ca_cert=ca_cert, + proxy=proxy, + pac=pac, + grpc_proxy=grpc_proxy, + ) self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace) self.nb_threads = nb_threads if nb_threads and nb_threads > 0: - self.threaded_scan = ThreadedScanning(self.scanoss_api, debug=debug, trace=trace, quiet=quiet, - nb_threads=nb_threads - ) + self.threaded_scan = ThreadedScanning( + self.scanoss_api, debug=debug, trace=trace, quiet=quiet, nb_threads=nb_threads + ) else: self.threaded_scan = None self.max_post_size = post_size * 1024 if post_size > 0 else MAX_POST_SIZE # Set the max post size (default 64k) @@ -157,7 +184,9 @@ def __init__( self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets self.scan_settings = scan_settings - self.post_processor = ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None + self.post_processor = ( + ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None + ) self._maybe_set_api_sbom() def _maybe_set_api_sbom(self): @@ -183,7 +212,6 @@ def __count_files_in_wfp_file(wfp_file: str): if WFP_FILE_START in line: count += 1 return count - @staticmethod def version_details() -> str: @@ -211,7 +239,7 @@ def __log_result(self, string, outfile=None): if not outfile and self.scan_output: outfile = self.scan_output if outfile: - with open(outfile, "a") as rf: + with open(outfile, 'a') as rf: rf.write(string + '\n') else: print(string) @@ -252,9 +280,15 @@ def is_dependency_scan(self): return True return False - def scan_folder_with_options(self, scan_dir: str, deps_file: str = None, file_map: dict = None, - dep_scope: SCOPE = None, dep_scope_include: str = None, - dep_scope_exclude: str = None) -> bool: + def scan_folder_with_options( + self, + scan_dir: str, + deps_file: str = None, + file_map: dict = None, + dep_scope: SCOPE = None, + dep_scope_include: str = None, + dep_scope_exclude: str = None, + ) -> bool: """ Scan the given folder for whatever scaning options that have been configured :param dep_scope_exclude: comma separated list of dependency scopes to exclude @@ -268,17 +302,23 @@ def scan_folder_with_options(self, scan_dir: str, deps_file: str = None, file_ma success = True if not scan_dir: - raise Exception(f"ERROR: Please specify a folder to scan") + raise Exception(f'ERROR: Please specify a folder to scan') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): - raise Exception(f"ERROR: Specified folder does not exist or is not a folder: {scan_dir}") + raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): - raise Exception(f"ERROR: No scan options defined to scan folder: {scan_dir}") + raise Exception(f'ERROR: No scan options defined to scan folder: {scan_dir}') if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') if self.is_dependency_scan(): - if not self.threaded_deps.run(what_to_scan=scan_dir, deps_file=deps_file, wait=False, dep_scope=dep_scope, - dep_scope_include= dep_scope_include, dep_scope_exclude=dep_scope_exclude): # Kick off a background dependency scan + if not self.threaded_deps.run( + what_to_scan=scan_dir, + deps_file=deps_file, + wait=False, + dep_scope=dep_scope, + dep_scope_include=dep_scope_include, + dep_scope_exclude=dep_scope_exclude, + ): # Kick off a background dependency scan success = False if self.is_file_or_snippet_scan(): if not self.scan_folder(scan_dir): @@ -302,16 +342,19 @@ def scan_folder(self, scan_dir: str) -> bool: if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') - file_filters = FileFilters(debug=self.debug, trace=self.trace, quiet=self.quiet, - scanoss_settings=self.scan_settings, - all_extensions=self.all_extensions, - all_folders=self.all_folders, - hidden_files_folders=self.hidden_files_folders, - skip_size=self.skip_size, - skip_folders=self.skip_folders, - skip_extensions=self.skip_extensions, - operation_type='scanning' - ) + file_filters = FileFilters( + debug=self.debug, + trace=self.trace, + quiet=self.quiet, + scanoss_settings=self.scan_settings, + all_extensions=self.all_extensions, + all_folders=self.all_folders, + hidden_files_folders=self.hidden_files_folders, + skip_size=self.skip_size, + skip_folders=self.skip_folders, + skip_extensions=self.skip_extensions, + operation_type='scanning', + ) self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None if not self.quiet and self.isatty: @@ -342,15 +385,15 @@ def scan_folder(self, scan_dir: str) -> bool: wfp_list.append(wfp) file_count += 1 if self.threaded_scan: - wfp_size = len(wfp.encode("utf-8")) + wfp_size = len(wfp.encode('utf-8')) # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 scan_block += wfp - scan_size = len(scan_block.encode("utf-8")) + scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: @@ -421,9 +464,7 @@ def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> b if self.is_dependency_scan(): self.print_msg('Retrieving dependency data...') if not self.threaded_deps.complete(): - self.print_stderr( - f'Warning: Dependency analysis ran into some trouble.' - ) + self.print_stderr(f'Warning: Dependency analysis ran into some trouble.') success = False dep_responses = self.threaded_deps.responses @@ -453,7 +494,7 @@ def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> b def _merge_scan_results( self, scan_responses: Optional[List], - dep_responses: Optional[Dict[str,Any]], + dep_responses: Optional[Dict[str, Any]], file_map: Optional[Dict[str, Any]], ) -> Dict[str, Any]: """Merge scan and dependency responses into a single dictionary""" @@ -466,10 +507,10 @@ def _merge_scan_results( response = self._deobfuscate_filenames(response, file_map) results.update(response) - dep_files = dep_responses.get("files", None) if dep_responses else None + dep_files = dep_responses.get('files', None) if dep_responses else None if dep_files: for dep_file in dep_files: - file = dep_file.pop("file", None) + file = dep_file.pop('file', None) if file: results[file] = [dep_file] @@ -486,8 +527,15 @@ def _deobfuscate_filenames(self, response: dict, file_map: dict) -> dict: deobfuscated[key] = value return deobfuscated - def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dict = None, dep_scope: SCOPE = None, - dep_scope_include: str = None, dep_scope_exclude: str = None) -> bool: + def scan_file_with_options( + self, + file: str, + deps_file: str = None, + file_map: dict = None, + dep_scope: SCOPE = None, + dep_scope_include: str = None, + dep_scope_exclude: str = None, + ) -> bool: """ Scan the given file for whatever scaning options that have been configured :param dep_scope: @@ -498,17 +546,23 @@ def scan_file_with_options(self, file: str, deps_file: str = None, file_map: dic """ success = True if not file: - raise Exception(f"ERROR: Please specify a file to scan") + raise Exception(f'ERROR: Please specify a file to scan') if not os.path.exists(file) or not os.path.isfile(file): - raise Exception(f"ERROR: Specified file does not exist or is not a file: {file}") + raise Exception(f'ERROR: Specified file does not exist or is not a file: {file}') if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): - raise Exception(f"ERROR: No scan options defined to scan file: {file}") + raise Exception(f'ERROR: No scan options defined to scan file: {file}') if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') if self.is_dependency_scan(): - if not self.threaded_deps.run(what_to_scan=file, deps_file=deps_file, wait=False, dep_scope=dep_scope, - dep_scope_include=dep_scope_include, dep_scope_exclude=dep_scope_exclude): # Kick off a background dependency scan + if not self.threaded_deps.run( + what_to_scan=file, + deps_file=deps_file, + wait=False, + dep_scope=dep_scope, + dep_scope_include=dep_scope_include, + dep_scope_exclude=dep_scope_exclude, + ): # Kick off a background dependency scan success = False if self.is_file_or_snippet_scan(): if not self.scan_file(file): @@ -529,9 +583,9 @@ def scan_file(self, file: str) -> bool: """ success = True if not file: - raise Exception(f"ERROR: Please specify a file to scan") + raise Exception(f'ERROR: Please specify a file to scan') if not os.path.exists(file) or not os.path.isfile(file): - raise Exception(f"ERROR: Specified files does not exist or is not a file: {file}") + raise Exception(f'ERROR: Specified files does not exist or is not a file: {file}') self.print_debug(f'Fingerprinting {file}...') wfp = self.winnowing.wfp_for_file(file, file) if wfp is not None and wfp != '': @@ -554,18 +608,21 @@ def scan_files(self, files: []) -> bool: """ success = True if not files: - raise Exception(f"ERROR: Please provide a non-empty list of filenames to scan") - - file_filters = FileFilters(debug=self.debug, trace=self.trace, quiet=self.quiet, - scanoss_settings=self.scan_settings, - all_extensions=self.all_extensions, - all_folders=self.all_folders, - hidden_files_folders=self.hidden_files_folders, - skip_size=self.skip_size, - skip_folders=self.skip_folders, - skip_extensions=self.skip_extensions, - operation_type='scanning' - ) + raise Exception(f'ERROR: Please provide a non-empty list of filenames to scan') + + file_filters = FileFilters( + debug=self.debug, + trace=self.trace, + quiet=self.quiet, + scanoss_settings=self.scan_settings, + all_extensions=self.all_extensions, + all_folders=self.all_folders, + hidden_files_folders=self.hidden_files_folders, + skip_size=self.skip_size, + skip_folders=self.skip_folders, + skip_extensions=self.skip_extensions, + operation_type='scanning', + ) spinner = None if not self.quiet and self.isatty: spinner = Spinner('Fingerprinting ') @@ -577,7 +634,7 @@ def scan_files(self, files: []) -> bool: file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False - + to_scan_files = file_filters.get_filtered_files_from_files(files) for file in to_scan_files: if self.threaded_scan and self.threaded_scan.stop_scanning(): @@ -617,7 +674,7 @@ def scan_files(self, files: []) -> bool: f'Warning: Some errors encounted while scanning. Results might be incomplete.' ) success = False - + # End for loop if self.threaded_scan and scan_block != '': self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted @@ -647,11 +704,13 @@ def scan_files_with_options(self, files: [], deps_file: str = None, file_map: di """ success = True if not files: - raise Exception(f"ERROR: Please specify a list of files to scan") + raise Exception(f'ERROR: Please specify a list of files to scan') if not self.is_file_or_snippet_scan(): - raise Exception(f"ERROR: file or snippet scan options have to be set to scan files: {files}") + raise Exception(f'ERROR: file or snippet scan options have to be set to scan files: {files}') if self.is_dependency_scan() or deps_file: - raise Exception(f"ERROR: The dependency scan option is currently not supported when scanning a list of files") + raise Exception( + f'ERROR: The dependency scan option is currently not supported when scanning a list of files' + ) if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') if self.is_file_or_snippet_scan(): @@ -672,9 +731,9 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: """ success = True if not filename: - raise Exception(f"ERROR: Please specify a filename to scan") + raise Exception(f'ERROR: Please specify a filename to scan') if not contents: - raise Exception(f"ERROR: Please specify a file contents to scan") + raise Exception(f'ERROR: Please specify a file contents to scan') self.print_debug(f'Fingerprinting {filename}...') wfp = self.winnowing.wfp_for_contents(filename, False, contents) @@ -700,7 +759,7 @@ def scan_wfp_file(self, file: str = None) -> bool: success = True wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): - raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") + raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}') file_count = Scanner.__count_files_in_wfp_file(wfp_file) cur_files = 0 cur_size = 0 @@ -709,7 +768,7 @@ def scan_wfp_file(self, file: str = None) -> bool: max_component = {'name': '', 'hits': 0} components = {} self.print_debug(f'Found {file_count} files to process.') - raw_output = "{\n" + raw_output = '{\n' file_print = '' bar = None if not self.quiet and self.isatty: @@ -720,7 +779,7 @@ def scan_wfp_file(self, file: str = None) -> bool: if line.startswith(WFP_FILE_START): if file_print: wfp += file_print # Store the WFP for the current file - cur_size = len(wfp.encode("utf-8")) + cur_size = len(wfp.encode('utf-8')) file_print = line # Start storing the next file cur_files += 1 batch_files += 1 @@ -729,8 +788,10 @@ def scan_wfp_file(self, file: str = None) -> bool: l_size = cur_size + len(file_print.encode('utf-8')) # Hit the max post size, so sending the current batch and continue processing if l_size >= self.max_post_size and wfp: - self.print_debug(f'Sending {batch_files} ({cur_files}) of' - f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.') + self.print_debug( + f'Sending {batch_files} ({cur_files}) of' + f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' + ) if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store @@ -738,7 +799,7 @@ def scan_wfp_file(self, file: str = None) -> bool: bar.next(batch_files) if scan_resp is not None: for key, value in scan_resp.items(): - raw_output += " \"%s\":%s," % (key, json.dumps(value, indent=2)) + raw_output += ' "%s":%s,' % (key, json.dumps(value, indent=2)) for v in value: if hasattr(v, 'get'): if v.get('id') != 'none': @@ -756,8 +817,10 @@ def scan_wfp_file(self, file: str = None) -> bool: if file_print: wfp += file_print # Store the WFP for the current file if wfp: - self.print_debug(f'Sending {batch_files} ({cur_files}) of' - f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.') + self.print_debug( + f'Sending {batch_files} ({cur_files}) of' + f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' + ) scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store if bar: bar.next(batch_files) @@ -765,13 +828,13 @@ def scan_wfp_file(self, file: str = None) -> bool: if scan_resp is not None: for key, value in scan_resp.items(): if first: - raw_output += " \"%s\":%s" % (key, json.dumps(value, indent=2)) + raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2)) first = False else: - raw_output += ",\n \"%s\":%s" % (key, json.dumps(value, indent=2)) + raw_output += ',\n "%s":%s' % (key, json.dumps(value, indent=2)) else: success = False - raw_output += "\n}" + raw_output += '\n}' if bar: bar.finish() if self.output_format == 'plain': @@ -802,10 +865,10 @@ def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) success = True wfp_file = wfp if wfp else self.wfp # If a WFP file is specified, use it, otherwise us the default if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): - raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") + raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}') if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): - raise Exception(f"ERROR: No scan options defined to scan WFP: {wfp}") + raise Exception(f'ERROR: No scan options defined to scan WFP: {wfp}') if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') @@ -829,7 +892,7 @@ def scan_wfp_file_threaded(self, file: str = None) -> bool: success = True wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): - raise Exception(f"ERROR: Specified WFP file does not exist or is not a file: {wfp_file}") + raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}') cur_size = 0 queue_size = 0 file_count = 0 # count all files fingerprinted @@ -842,7 +905,7 @@ def scan_wfp_file_threaded(self, file: str = None) -> bool: if line.startswith(WFP_FILE_START): if scan_block: wfp += scan_block # Store the WFP for the current file - cur_size = len(wfp.encode("utf-8")) + cur_size = len(wfp.encode('utf-8')) scan_block = line # Start storing the next file file_count += 1 wfp_file_count += 1 @@ -861,7 +924,8 @@ def scan_wfp_file_threaded(self, file: str = None) -> bool: scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( - f'Warning: Some errors encounted while scanning. Results might be incomplete.') + f'Warning: Some errors encounted while scanning. Results might be incomplete.' + ) success = False # End for loop if scan_block: @@ -884,15 +948,15 @@ def scan_wfp(self, wfp: str) -> bool: """ success = True if not wfp: - raise Exception(f"ERROR: Please specify a WFP to scan") - raw_output = "{\n" + raise Exception(f'ERROR: Please specify a WFP to scan') + raw_output = '{\n' scan_resp = self.scanoss_api.scan(wfp) if scan_resp is not None: for key, value in scan_resp.items(): - raw_output += " \"%s\":%s" % (key, json.dumps(value, indent=2)) + raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2)) else: success = False - raw_output += "\n}" + raw_output += '\n}' if self.output_format == 'plain': self.__log_result(raw_output) elif self.output_format == 'cyclonedx': @@ -920,9 +984,9 @@ def wfp_contents(self, filename: str, contents: bytes, wfp_file: str = None): :return: """ if not filename: - raise Exception(f"ERROR: Please specify a filename to scan") + raise Exception(f'ERROR: Please specify a filename to scan') if not contents: - raise Exception(f"ERROR: Please specify a file contents to scan") + raise Exception(f'ERROR: Please specify a file contents to scan') self.print_debug(f'Fingerprinting {filename}...') wfp = self.winnowing.wfp_for_contents(filename, False, contents) @@ -941,9 +1005,9 @@ def wfp_file(self, scan_file: str, wfp_file: str = None): Fingerprint the specified file """ if not scan_file: - raise Exception(f"ERROR: Please specify a file to fingerprint") + raise Exception(f'ERROR: Please specify a file to fingerprint') if not os.path.exists(scan_file) or not os.path.isfile(scan_file): - raise Exception(f"ERROR: Specified file does not exist or is not a file: {scan_file}") + raise Exception(f'ERROR: Specified file does not exist or is not a file: {scan_file}') self.print_debug(f'Fingerprinting {scan_file}...') wfp = self.winnowing.wfp_for_file(scan_file, scan_file) @@ -965,16 +1029,19 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): raise Exception(f'ERROR: Please specify a folder to fingerprint') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') - file_filters = FileFilters(debug=self.debug, trace=self.trace, quiet=self.quiet, - scanoss_settings=self.scan_settings, - all_extensions=self.all_extensions, - all_folders=self.all_folders, - hidden_files_folders=self.hidden_files_folders, - skip_size=self.skip_size, - skip_folders=self.skip_folders, - skip_extensions=self.skip_extensions, - operation_type='scanning' - ) + file_filters = FileFilters( + debug=self.debug, + trace=self.trace, + quiet=self.quiet, + scanoss_settings=self.scan_settings, + all_extensions=self.all_extensions, + all_folders=self.all_folders, + hidden_files_folders=self.hidden_files_folders, + skip_size=self.skip_size, + skip_folders=self.skip_folders, + skip_extensions=self.skip_extensions, + operation_type='scanning', + ) wfps = '' self.print_msg(f'Searching {scan_dir} for files to fingerprint...') spinner = None @@ -1000,6 +1067,7 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): else: Scanner.print_stderr(f'Warning: No files found to fingerprint in folder: {scan_dir}') + # # End of ScanOSS Class # diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index cec4449a..0903f98f 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -119,7 +119,9 @@ def load_json_file(self, filepath: 'str | None' = None, scan_root: 'str | None' result = validate_json_file(json_file) if not result.is_valid: if result.error_code == JSON_ERROR_FILE_NOT_FOUND or result.error_code == JSON_ERROR_FILE_EMPTY: - self.print_msg(f'WARNING: The supplied settings file "{filepath}" was not found or is empty. Skipping...') + self.print_msg( + f'WARNING: The supplied settings file "{filepath}" was not found or is empty. Skipping...' + ) return self else: raise ScanossSettingsError(f'Problem with settings file. {result.error}') @@ -234,8 +236,8 @@ def _get_sbom_assets(self): replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace())) self.print_debug( f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n" - f"From Include list: {[entry['purl'] for entry in include_bom_entries]} \n" - f"From Replace list: {[entry['purl'] for entry in replace_bom_entries]} \n" + f'From Include list: {[entry["purl"] for entry in include_bom_entries]} \n' + f'From Replace list: {[entry["purl"] for entry in replace_bom_entries]} \n' ) return include_bom_entries + replace_bom_entries return self.normalize_bom_entries(self.get_bom_remove()) diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 3a0643d6..fc5d3522 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2022, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2022, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import logging import os import sys @@ -39,10 +40,10 @@ from . import __version__ -DEFAULT_URL = "https://api.osskb.org/scan/direct" # default free service URL -DEFAULT_URL2 = "https://api.scanoss.com/scan/direct" # default premium service URL -SCANOSS_SCAN_URL = os.environ.get("SCANOSS_SCAN_URL") if os.environ.get("SCANOSS_SCAN_URL") else DEFAULT_URL -SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' +DEFAULT_URL = 'https://api.osskb.org/scan/direct' # default free service URL +DEFAULT_URL2 = 'https://api.scanoss.com/scan/direct' # default premium service URL +SCANOSS_SCAN_URL = os.environ.get('SCANOSS_SCAN_URL') if os.environ.get('SCANOSS_SCAN_URL') else DEFAULT_URL +SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else '' class ScanossApi(ScanossBase): @@ -51,10 +52,23 @@ class ScanossApi(ScanossBase): Currently support posting scan requests to the SCANOSS streaming API """ - def __init__(self, scan_format: str = None, flags: str = None, - url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - timeout: int = 180, ver_details: str = None, ignore_cert_errors: bool = False, - proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5): + def __init__( + self, + scan_format: str = None, + flags: str = None, + url: str = None, + api_key: str = None, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + timeout: int = 180, + ver_details: str = None, + ignore_cert_errors: bool = False, + proxy: str = None, + ca_cert: str = None, + pac: PACFile = None, + retry: int = 5, + ): """ Initialise the SCANOSS API :param scan_format: Scan format (default plain) @@ -74,7 +88,7 @@ def __init__(self, scan_format: str = None, flags: str = None, super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_SCAN_URL self.api_key = api_key if api_key else SCANOSS_API_KEY - if self.api_key and not url and not os.environ.get("SCANOSS_SCAN_URL"): + if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.sbom = None self.scan_format = scan_format if scan_format else 'plain' @@ -108,7 +122,7 @@ def __init__(self, scan_format: str = None, flags: str = None, self.verify = ca_cert self.session.verify = ca_cert self.proxies = {'https': proxy, 'http': proxy} if proxy else None - if self. proxies: + if self.proxies: self.session.proxies = self.proxies def scan(self, wfp: str, context: str = None, scan_id: int = None): @@ -122,16 +136,16 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): request_id = str(uuid.uuid4()) form_data = {} if self.sbom: - form_data['type'] = self.sbom.get("scan_type") - form_data['assets'] = self.sbom.get("assets") + form_data['type'] = self.sbom.get('scan_type') + form_data['assets'] = self.sbom.get('assets') if self.scan_format: form_data['format'] = self.scan_format if self.flags: form_data['flags'] = self.flags if context: form_data['context'] = context - - scan_files = {'file': ("%s.wfp" % request_id, wfp)} + + scan_files = {'file': ('%s.wfp' % request_id, wfp)} headers = self.headers headers['x-request-id'] = request_id # send a unique request id for each post r = None @@ -140,75 +154,87 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): retry += 1 try: r = None - r = self.session.post(self.url, files=scan_files, data=form_data, headers=self.headers, - timeout=self.timeout - ) + r = self.session.post( + self.url, files=scan_files, data=form_data, headers=self.headers, timeout=self.timeout + ) except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e: self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') - raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e + raise Exception(f'ERROR: The SCANOSS API request failed for {self.url}') from e except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: if retry > self.retry_limit: # Timed out retry_limit or more times, fail self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data ({request_id}) - {e}: {scan_files}') - raise Exception(f"ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for" - f" {self.url}") from e + raise Exception( + f'ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for {self.url}' + ) from e else: self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...') time.sleep(5) except Exception as e: - self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) - {e}:' - f' {scan_files}') - raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e + self.print_stderr( + f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) - {e}: {scan_files}' + ) + raise Exception(f'ERROR: The SCANOSS API request failed for {self.url}') from e else: if r is None: if retry > self.retry_limit: # No response retry_limit or more times, fail self.save_bad_req_wfp(scan_files, request_id, scan_id) - raise Exception(f"ERROR: The SCANOSS API request ({request_id}) response object is empty " - f"for {self.url}") + raise Exception( + f'ERROR: The SCANOSS API request ({request_id}) response object is empty for {self.url}' + ) else: self.print_stderr(f'Warning: No response received from {self.url}. Retrying...') time.sleep(5) elif r.status_code == 503: # Service limits have most likely been reached - self.print_stderr(f'ERROR: SCANOSS API rejected the scan request ({request_id}) due to ' - f'service limits being exceeded') + self.print_stderr( + f'ERROR: SCANOSS API rejected the scan request ({request_id}) due to ' + f'service limits being exceeded' + ) self.print_stderr(f'ERROR: Details: {r.text.strip()}') - raise Exception(f"ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected " - f"for {self.url} due to service limits being exceeded.") + raise Exception( + f'ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected ' + f'for {self.url} due to service limits being exceeded.' + ) elif r.status_code >= 400: if retry > self.retry_limit: # No response retry_limit or more times, fail self.save_bad_req_wfp(scan_files, request_id, scan_id) raise Exception( - f"ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, " - f"{r.text.strip()}") + f'ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, ' + f'{r.text.strip()}' + ) else: self.save_bad_req_wfp(scan_files, request_id, scan_id) - self.print_stderr(f'Warning: Error response code {r.status_code} ({r.text.strip()}) from ' - f'{self.url}. Retrying...') + self.print_stderr( + f'Warning: Error response code {r.status_code} ({r.text.strip()}) from ' + f'{self.url}. Retrying...' + ) time.sleep(5) else: break # Valid response, break out of the retry loop # End of while loop if r is None: self.save_bad_req_wfp(scan_files, request_id, scan_id) - raise Exception(f"ERROR: The SCANOSS API request response object is empty for {self.url}") + raise Exception(f'ERROR: The SCANOSS API request response object is empty for {self.url}') try: if 'xml' in self.scan_format: # TODO remove XML parsing option? return r.text json_resp = r.json() return json_resp except (JSONDecodeError, Exception) as e: - self.print_stderr(f'ERROR: The SCANOSS API returned an invalid JSON ' - f'({e.__class__.__name__} - {request_id}): {e}') + self.print_stderr( + f'ERROR: The SCANOSS API returned an invalid JSON ({e.__class__.__name__} - {request_id}): {e}' + ) bad_json_file = f'bad_json-{scan_id}-{request_id}.txt' if scan_id else f'bad_json-{request_id}.txt' self.print_stderr(f'Ignoring result. Please look in "{bad_json_file}" for more details.') try: with open(bad_json_file, 'w') as f: - f.write(f"---Request ID Begin---\n{request_id}\n---Request ID End---\n") - f.write(f"---WFP Begin---\n{scan_files}\n---WFP End---\n---Bad JSON Begin---\n") + f.write(f'---Request ID Begin---\n{request_id}\n---Request ID End---\n') + f.write(f'---WFP Begin---\n{scan_files}\n---WFP End---\n---Bad JSON Begin---\n') f.write(r.text) - f.write("---Bad JSON End---\n") + f.write('---Bad JSON End---\n') except Exception as ee: - self.print_stderr(f'Warning: Issue writing bad json file - {bad_json_file} ({ee.__class__.__name__}):' - f' {ee}') + self.print_stderr( + f'Warning: Issue writing bad json file - {bad_json_file} ({ee.__class__.__name__}): {ee}' + ) return None def save_bad_req_wfp(self, scan_files, request_id, scan_id): @@ -220,19 +246,22 @@ def save_bad_req_wfp(self, scan_files, request_id, scan_id): """ bad_req_file = f'bad_request-{scan_id}-{request_id}.txt' if scan_id else f'bad_request-{request_id}.txt' try: - self.print_stderr(f'No response object returned from API. Please look in "{bad_req_file}" for the ' - f'offending WFP.') + self.print_stderr( + f'No response object returned from API. Please look in "{bad_req_file}" for the offending WFP.' + ) with open(bad_req_file, 'w') as f: - f.write(f"---Request ID Begin---\n{request_id}\n---Request ID End---\n") - f.write(f"---WFP Begin---\n{scan_files}\n---WFP End---\n") + f.write(f'---Request ID Begin---\n{request_id}\n---Request ID End---\n') + f.write(f'---WFP Begin---\n{scan_files}\n---WFP End---\n') except Exception as ee: - self.print_stderr(f'Warning: Issue writing bad request file - {bad_req_file} ({ee.__class__.__name__}):' - f' {ee}') - + self.print_stderr( + f'Warning: Issue writing bad request file - {bad_req_file} ({ee.__class__.__name__}): {ee}' + ) + def set_sbom(self, sbom): self.sbom = sbom return self + # # End of ScanossApi Class # diff --git a/src/scanoss/scanossbase.py b/src/scanoss/scanossbase.py index 974e2bf2..c5f79c4a 100644 --- a/src/scanoss/scanossbase.py +++ b/src/scanoss/scanossbase.py @@ -1,25 +1,25 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ import sys @@ -85,7 +85,7 @@ def print_to_file_or_stdout(self, msg: str, file: str = None): Print message to file if provided or stdout """ if file: - with open(file, "w") as f: + with open(file, 'w') as f: f.write(msg) else: self.print_stdout(msg) @@ -95,7 +95,7 @@ def print_to_file_or_stderr(self, msg: str, file: str = None): Print message to file if provided or stderr """ if file: - with open(file, "w") as f: + with open(file, 'w') as f: f.write(msg) else: self.print_stderr(msg) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index b8475233..0e21aeaa 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -1,25 +1,25 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ import os @@ -44,17 +44,20 @@ from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse -from .api.components.v2.scanoss_components_pb2 import (CompSearchRequest, CompSearchResponse, - CompVersionRequest, CompVersionResponse) +from .api.components.v2.scanoss_components_pb2 import ( + CompSearchRequest, + CompSearchResponse, + CompVersionRequest, + CompVersionResponse, +) from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse - from .scanossbase import ScanossBase from . import __version__ -DEFAULT_URL = "https://api.osskb.org" # default free service URL -DEFAULT_URL2 = "https://api.scanoss.com" # default premium service URL -SCANOSS_GRPC_URL = os.environ.get("SCANOSS_GRPC_URL") if os.environ.get("SCANOSS_GRPC_URL") else DEFAULT_URL -SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else '' +DEFAULT_URL = 'https://api.osskb.org' # default free service URL +DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL +SCANOSS_GRPC_URL = os.environ.get('SCANOSS_GRPC_URL') if os.environ.get('SCANOSS_GRPC_URL') else DEFAULT_URL +SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else '' class ScanossGrpc(ScanossBase): @@ -62,9 +65,20 @@ class ScanossGrpc(ScanossBase): Client for gRPC functionality """ - def __init__(self, url: str = None, debug: bool = False, trace: bool = False, quiet: bool = False, - ca_cert: str = None, api_key: str = None, ver_details: str = None, timeout: int = 600, - proxy: str = None, grpc_proxy: str = None, pac: PACFile = None): + def __init__( + self, + url: str = None, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + ca_cert: str = None, + api_key: str = None, + ver_details: str = None, + timeout: int = 600, + proxy: str = None, + grpc_proxy: str = None, + pac: PACFile = None, + ): """ :param url: @@ -83,7 +97,7 @@ def __init__(self, url: str = None, debug: bool = False, trace: bool = False, qu super().__init__(debug, trace, quiet) self.url = url if url else SCANOSS_GRPC_URL self.api_key = api_key if api_key else SCANOSS_API_KEY - if self.api_key and not url and not os.environ.get("SCANOSS_GRPC_URL"): + if self.api_key and not url and not os.environ.get('SCANOSS_GRPC_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.url = self.url.lower() self.orig_url = self.url # Used for proxy lookup @@ -148,8 +162,9 @@ def deps_echo(self, message: str = 'Hello there!') -> str: metadata.append(('x-request-id', request_id)) # Set a Request ID resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: # self.print_stderr(f'resp: {resp} - call: {call}') # response_id = "" @@ -181,8 +196,9 @@ def crypto_echo(self, message: str = 'Hello there!') -> str: metadata.append(('x-request-id', request_id)) # Set a Request ID resp = self.crypto_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: return resp.message @@ -211,7 +227,7 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: request_id = str(uuid.uuid4()) resp: DependencyResponse try: - files_json = dependencies.get("files") + files_json = dependencies.get('files') if files_json is None or len(files_json) == 0: self.print_stderr(f'ERROR: No dependency data supplied to send to gRPC service.') return None @@ -222,8 +238,9 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...') resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: if not self._check_status_response(resp.status, request_id): @@ -249,8 +266,9 @@ def get_crypto_json(self, purls: dict) -> dict: self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...') resp = self.crypto_stub.GetAlgorithms(request, metadata=metadata, timeout=self.timeout) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: if not self._check_status_response(resp.status, request_id): @@ -278,8 +296,9 @@ def get_vulnerabilities_json(self, purls: dict) -> dict: self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...') resp = self.vuln_stub.GetVulnerabilities(request, metadata=metadata, timeout=self.timeout) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: if not self._check_status_response(resp.status, request_id): @@ -307,8 +326,9 @@ def get_semgrep_json(self, purls: dict) -> dict: self.print_debug(f'Sending semgrep data for decoration (rqId: {request_id})...') resp = self.semgrep_stub.GetIssues(request, metadata=metadata, timeout=self.timeout) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: if not self._check_status_response(resp.status, request_id): @@ -336,8 +356,9 @@ def search_components_json(self, search: dict) -> dict: self.print_debug(f'Sending component search data (rqId: {request_id})...') resp = self.comp_search_stub.SearchComponents(request, metadata=metadata, timeout=self.timeout) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: if not self._check_status_response(resp.status, request_id): @@ -365,8 +386,9 @@ def get_component_versions_json(self, search: dict) -> dict: self.print_debug(f'Sending component version data (rqId: {request_id})...') resp = self.comp_search_stub.GetComponentVersions(request, metadata=metadata, timeout=self.timeout) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: if not self._check_status_response(resp.status, request_id): @@ -394,7 +416,7 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st msg = "Succeeded with warnings" ret_val = True # No need to fail as it succeeded with warnings elif status_code == 3: - msg = "Failed with warnings" + msg = 'Failed with warnings' self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}') return ret_val return True @@ -407,19 +429,19 @@ def _get_proxy_config(self): """ if self.grpc_proxy: self.print_debug(f'Setting GRPC (grpc_proxy) proxy...') - os.environ["grpc_proxy"] = self.grpc_proxy + os.environ['grpc_proxy'] = self.grpc_proxy elif self.proxy: self.print_debug(f'Setting GRPC (http_proxy/https_proxy) proxies...') - os.environ["http_proxy"] = self.proxy - os.environ["https_proxy"] = self.proxy + os.environ['http_proxy'] = self.proxy + os.environ['https_proxy'] = self.proxy elif self.pac: self.print_debug(f'Attempting to get GRPC proxy details from PAC for {self.orig_url}...') resolver = ProxyResolver(self.pac) proxies = resolver.get_proxy_for_requests(self.orig_url) if proxies: self.print_trace(f'Setting proxies: {proxies}') - os.environ["http_proxy"] = proxies.get("http") or "" - os.environ["https_proxy"] = proxies.get("https") or "" + os.environ['http_proxy'] = proxies.get('http') or '' + os.environ['https_proxy'] = proxies.get('https') or '' def get_provenance_json(self, purls: dict) -> dict: """ diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index be35c5a4..4dd6f85d 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -207,8 +207,9 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di return result - def _should_replace_result(self, result_path: str, result: dict, to_replace_entries: List[BomEntry] - ) -> Tuple[bool, str]: + def _should_replace_result( + self, result_path: str, result: dict, to_replace_entries: List[BomEntry] + ) -> Tuple[bool, str]: """ Check if a result should be replaced based on the SCANOSS settings @@ -278,14 +279,16 @@ def _print_message(self, result_path: str, result_purls: List[str], bom_entry: B :return: """ message = ( - f"{_get_match_type_message(result_path, bom_entry, action)} \n" - f"Details:\n" - f" - PURLs: {', '.join(result_purls)}\n" + f'{_get_match_type_message(result_path, bom_entry, action)} \n' + f'Details:\n' + f' - PURLs: {", ".join(result_purls)}\n' f" - Path: '{result_path}'\n" ) if action == 'Replacing': message += f" - {action} with '{bom_entry.get('replace_with')}'" self.print_debug(message) + + # # End of ScanPostProcessor Class -# \ No newline at end of file +# diff --git a/src/scanoss/scantype.py b/src/scanoss/scantype.py index 67effb41..9e617c66 100644 --- a/src/scanoss/scantype.py +++ b/src/scanoss/scantype.py @@ -1,25 +1,25 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ from enum import Enum @@ -29,6 +29,7 @@ class ScanType(Enum): """ Octal Enum class describing all the scanning options """ + SCAN_FILES = 1 SCAN_SNIPPETS = 2 SCAN_DEPENDENCIES = 4 diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index d72ebc80..b8bf5c6b 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os.path import sys @@ -77,18 +78,18 @@ def parse(self, data: json): file_details = data.get(f) # print(f'File: {f}: {file_details}\n') for d in file_details: - id_details = d.get("id") + id_details = d.get('id') if not id_details or id_details == 'none': # Ignore files with no ids continue purl = None if id_details == 'dependency': # Process dependency data - dependencies = d.get("dependencies") + dependencies = d.get('dependencies') if not dependencies: self.print_stderr(f'Warning: No Dependencies found for {f}: {file_details}') continue for deps in dependencies: # print(f'File: {f} Deps: {deps}') - purl = deps.get("purl") + purl = deps.get('purl') if not purl: self.print_stderr(f'Warning: No PURL found for {f}: {deps}') continue @@ -103,7 +104,7 @@ def parse(self, data: json): if licenses: dc = [] for lic in licenses: - name = lic.get("name") + name = lic.get('name') if name not in dc: # Only save the license name once fdl.append({'id': name}) dc.append(name) @@ -132,7 +133,7 @@ def parse(self, data: json): if licenses: dc = [] for lic in licenses: - name = lic.get("name") + name = lic.get('name') if name not in dc: # Only save the license name once fdl.append({'id': name}) dc.append(name) @@ -175,7 +176,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: # pip3 install jsonschema # jsonschema -i spdxlite.json <(curl https://raw.githubusercontent.com/spdx/spdx-spec/v2.2/schemas/spdx-schema.json) # Validation can also be done online here: https://tools.spdx.org/app/validate/ - now = datetime.datetime.utcnow() # TODO replace with recommended format + now = datetime.datetime.utcnow() # TODO replace with recommended format md5hex = hashlib.md5(f'{raw_data}-{now}'.encode('utf-8')).hexdigest() data = { 'spdxVersion': 'SPDX-2.2', @@ -184,12 +185,12 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: 'name': 'SCANOSS-SBOM', 'creationInfo': { 'created': now.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'creators': [f'Tool: SCANOSS-PY: {__version__}', f'Person: {getpass.getuser()}'] + 'creators': [f'Tool: SCANOSS-PY: {__version__}', f'Person: {getpass.getuser()}'], }, 'documentNamespace': f'https://spdx.org/spdxdocs/scanoss-py-{__version__}-{md5hex}', 'documentDescribes': [], 'hasExtractedLicensingInfos': [], - 'packages': [] + 'packages': [], } lic_refs = set() # Hash Set of non-SPDX license references for purl in raw_data: @@ -215,27 +216,27 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: comp_ver = comp.get('version') purl_ver = f'{purl}@{comp_ver}' vendor = comp.get('vendor', 'NOASSERTION') - supplier = f"Organization: {vendor}" if vendor != 'NOASSERTION' else vendor + supplier = f'Organization: {vendor}' if vendor != 'NOASSERTION' else vendor purl_hash = hashlib.md5(f'{purl_ver}'.encode('utf-8')).hexdigest() purl_spdx = f'SPDXRef-{purl_hash}' data['documentDescribes'].append(purl_spdx) - data['packages'].append({ - 'name': comp_name, - 'SPDXID': purl_spdx, - 'versionInfo': comp_ver, - 'downloadLocation': 'NOASSERTION', # TODO Add actual download location - 'homepage': comp.get('url', ''), - 'licenseDeclared': lic_text, - 'licenseConcluded': 'NOASSERTION', - 'filesAnalyzed': False, - 'copyrightText': 'NOASSERTION', - 'supplier': supplier, - 'externalRefs': [{ - 'referenceCategory': 'PACKAGE-MANAGER', - 'referenceLocator': purl_ver, - 'referenceType': 'purl' - }] - }) + data['packages'].append( + { + 'name': comp_name, + 'SPDXID': purl_spdx, + 'versionInfo': comp_ver, + 'downloadLocation': 'NOASSERTION', # TODO Add actual download location + 'homepage': comp.get('url', ''), + 'licenseDeclared': lic_text, + 'licenseConcluded': 'NOASSERTION', + 'filesAnalyzed': False, + 'copyrightText': 'NOASSERTION', + 'supplier': supplier, + 'externalRefs': [ + {'referenceCategory': 'PACKAGE-MANAGER', 'referenceLocator': purl_ver, 'referenceType': 'purl'} + ], + } + ) # End purls for loop for lic_ref in lic_refs: # Insert all the non-SPDX license references source = '' @@ -247,12 +248,14 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: name = lic_ref name = name.replace('-', ' ') source = f' by {source}.' if source else '.' - data['hasExtractedLicensingInfos'].append({ - 'licenseId': lic_ref, - 'name': name, - 'extractedText': 'Detected license, please review component source code.', - 'comment': f'Detected license{source}' - }) + data['hasExtractedLicensingInfos'].append( + { + 'licenseId': lic_ref, + 'name': name, + 'extractedText': 'Detected license, please review component source code.', + 'comment': f'Detected license{source}', + } + ) # End license refs for loop file = sys.stdout if not output_file and self.output_file: @@ -352,6 +355,8 @@ def get_spdx_license_id(self, lic_name: str) -> str: return lic_id self.print_debug(f'Warning: Failed to find valid SPDX license identifier for: {lic_name}') return None + + # # End of SpdxLite Class # diff --git a/src/scanoss/threadeddependencies.py b/src/scanoss/threadeddependencies.py index 036c6831..b2cff5d1 100644 --- a/src/scanoss/threadeddependencies.py +++ b/src/scanoss/threadeddependencies.py @@ -1,25 +1,25 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ import threading @@ -33,9 +33,21 @@ from .scanossbase import ScanossBase from .scanossgrpc import ScanossGrpc -DEP_FILE_PREFIX = "file=" # Default prefix to signify an existing parsed dependency file +DEP_FILE_PREFIX = 'file=' # Default prefix to signify an existing parsed dependency file -DEV_DEPENDENCIES = { "dev", "test", "development", "provided", "runtime", "devDependencies", "dev-dependencies", "testImplementation", "testCompile", "Test", "require-dev" } +DEV_DEPENDENCIES = { + 'dev', + 'test', + 'development', + 'provided', + 'runtime', + 'devDependencies', + 'dev-dependencies', + 'testImplementation', + 'testCompile', + 'Test', + 'require-dev', +} # Define an enum class @@ -46,17 +58,21 @@ class SCOPE(Enum): @dataclass class ThreadedDependencies(ScanossBase): - """ + """ """ - """ inputs: queue.Queue = queue.Queue() output: queue.Queue = queue.Queue() - def __init__(self, sc_deps: ScancodeDeps, grpc_api: ScanossGrpc, what_to_scan: str = None, debug: bool = False, - trace: bool = False, quiet: bool = False) -> None: - """ - - """ + def __init__( + self, + sc_deps: ScancodeDeps, + grpc_api: ScanossGrpc, + what_to_scan: str = None, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + ) -> None: + """ """ super().__init__(debug, trace, quiet) self.sc_deps = sc_deps self.grpc_api = grpc_api @@ -76,8 +92,15 @@ def responses(self) -> Dict: return resp return None - def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True, dep_scope: SCOPE = None, - dep_scope_include: str = None, dep_scope_exclude: str = None) -> bool: + def run( + self, + what_to_scan: str = None, + deps_file: str = None, + wait: bool = True, + dep_scope: SCOPE = None, + dep_scope_include: str = None, + dep_scope_exclude: str = None, + ) -> bool: """ Initiate a background scan for the specified file/dir :param dep_scope_exclude: comma separated list of dependency scopes to exclude @@ -91,23 +114,25 @@ def run(self, what_to_scan: str = None, deps_file: str = None, wait: bool = True what_to_scan = what_to_scan if what_to_scan else self.what_to_scan self._errors = False try: - if deps_file: # Decorate the given dependencies file + if deps_file: # Decorate the given dependencies file self.print_msg(f'Decorating {deps_file} dependencies...') - self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}') # Add to queue and have parent wait on it - else: # Search for dependencies to decorate + self.inputs.put(f'{DEP_FILE_PREFIX}{deps_file}') # Add to queue and have parent wait on it + else: # Search for dependencies to decorate self.print_msg(f'Searching {what_to_scan} for dependencies...') self.inputs.put(what_to_scan) # Add to queue and have parent wait on it - self._thread = threading.Thread(target=self.scan_dependencies(dep_scope, dep_scope_include, dep_scope_exclude), daemon=True) + self._thread = threading.Thread( + target=self.scan_dependencies(dep_scope, dep_scope_include, dep_scope_exclude), daemon=True + ) self._thread.start() except Exception as e: self.print_stderr(f'ERROR: Problem running threaded dependencies: {e}') self._errors = True - if wait and not self._errors: # Wait for all inputs to complete + if wait and not self._errors: # Wait for all inputs to complete self.complete() return False if self._errors else True - def filter_dependencies(self,deps ,filter_dep)-> json: + def filter_dependencies(self, deps, filter_dep) -> json: files = deps.get('files', []) # Iterate over files and their purls for file in files: @@ -120,52 +145,56 @@ def filter_dependencies(self,deps ,filter_dep)-> json: ] # End of for loop - return { - 'files': [ - file for file in deps.get('files', []) - if file.get('purls') - ] - } + return {'files': [file for file in deps.get('files', []) if file.get('purls')]} - def filter_dependencies_by_scopes(self,deps: json, dep_scope: SCOPE = None, dep_scope_include: str = None, - dep_scope_exclude: str = None) -> json: + def filter_dependencies_by_scopes( + self, deps: json, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None + ) -> json: # Predefined set of scopes to filter # Include all scopes - include_all = (dep_scope is None or dep_scope == "") and dep_scope_include is None and dep_scope_exclude is None + include_all = (dep_scope is None or dep_scope == '') and dep_scope_include is None and dep_scope_exclude is None ## All dependencies, remove scope key if include_all: - return self.filter_dependencies(deps, lambda purl:True) + return self.filter_dependencies(deps, lambda purl: True) # Use default list of scopes if a custom list is not set - if (dep_scope is not None and dep_scope != "") and dep_scope_include is None and dep_scope_exclude is None: - return self.filter_dependencies(deps, lambda purl: (dep_scope == SCOPE.PRODUCTION and purl not in DEV_DEPENDENCIES) or - dep_scope == SCOPE.DEVELOPMENT and purl in DEV_DEPENDENCIES) + if (dep_scope is not None and dep_scope != '') and dep_scope_include is None and dep_scope_exclude is None: + return self.filter_dependencies( + deps, + lambda purl: (dep_scope == SCOPE.PRODUCTION and purl not in DEV_DEPENDENCIES) + or dep_scope == SCOPE.DEVELOPMENT + and purl in DEV_DEPENDENCIES, + ) - if ((dep_scope_include is not None and dep_scope_include != "") - or dep_scope_exclude is not None and dep_scope_exclude != ""): + if ( + (dep_scope_include is not None and dep_scope_include != '') + or dep_scope_exclude is not None + and dep_scope_exclude != '' + ): # Create sets from comma-separated strings, if provided exclude = set(dep_scope_exclude.split(',')) if dep_scope_exclude else set() include = set(dep_scope_include.split(',')) if dep_scope_include else set() # Define a lambda function that checks the inclusion/exclusion logic return self.filter_dependencies( - deps, - lambda purl: (exclude and purl not in exclude) or (not exclude and purl in include) + deps, lambda purl: (exclude and purl not in exclude) or (not exclude and purl in include) ) - def scan_dependencies(self, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None) -> None: + def scan_dependencies( + self, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None + ) -> None: """ Scan for dependencies from the given file/dir or from an input file (from the input queue). """ current_thread = threading.get_ident() self.print_trace(f'Starting dependency worker {current_thread}...') try: - what_to_scan = self.inputs.get(timeout=5) # Begin processing the dependency request + what_to_scan = self.inputs.get(timeout=5) # Begin processing the dependency request deps = None - if what_to_scan.startswith(DEP_FILE_PREFIX): # We have a pre-parsed dependency file, load it + if what_to_scan.startswith(DEP_FILE_PREFIX): # We have a pre-parsed dependency file, load it deps = self.sc_deps.load_from_file(what_to_scan.strip(DEP_FILE_PREFIX)) - else: # Search the file/folder for dependency files to parse + else: # Search the file/folder for dependency files to parse if not self.sc_deps.run_scan(what_to_scan=what_to_scan): self._errors = True else: @@ -176,13 +205,13 @@ def scan_dependencies(self, dep_scope: SCOPE = None, dep_scope_include: str = No self.print_debug(f"Including dependencies with '{dep_scope_include.split(',')}' scopes") if dep_scope_exclude is not None: self.print_debug(f"Excluding dependencies with '{dep_scope_exclude.split(',')}' scopes") - deps = self.filter_dependencies_by_scopes(deps, dep_scope,dep_scope_include, dep_scope_exclude) + deps = self.filter_dependencies_by_scopes(deps, dep_scope, dep_scope_include, dep_scope_exclude) if not self._errors: if deps is None: self.print_stderr(f'Problem searching for dependencies for: {what_to_scan}') self._errors = True - elif not deps or len(deps.get("files", [])) == 0: + elif not deps or len(deps.get('files', [])) == 0: self.print_debug(f'No dependencies found to decorate for: {what_to_scan}') else: decorated_deps = self.grpc_api.get_dependencies(deps) @@ -210,6 +239,7 @@ def complete(self) -> bool: self._errors = True return True if not self._errors else False + # # End of ThreadedDependencies Class # diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index 26fbe27e..da1a180e 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import os import sys import threading @@ -34,8 +35,10 @@ from .scanossapi import ScanossApi from .scanossbase import ScanossBase -WFP_FILE_START = "file=" -MAX_ALLOWED_THREADS = int(os.environ.get("SCANOSS_MAX_ALLOWED_THREADS")) if os.environ.get("SCANOSS_MAX_ALLOWED_THREADS") else 30 +WFP_FILE_START = 'file=' +MAX_ALLOWED_THREADS = ( + int(os.environ.get('SCANOSS_MAX_ALLOWED_THREADS')) if os.environ.get('SCANOSS_MAX_ALLOWED_THREADS') else 30 +) @dataclass @@ -45,13 +48,14 @@ class ThreadedScanning(ScanossBase): WFP scan requests are loaded into the input queue. Multiple threads pull messages off this queue, process the request and put the results into an output queue """ + inputs: queue.Queue = queue.Queue() output: queue.Queue = queue.Queue() bar: Bar = None - def __init__(self, scanapi: ScanossApi, debug: bool = False, trace: bool = False, quiet: bool = False, - nb_threads: int = 5 - ) -> None: + def __init__( + self, scanapi: ScanossApi, debug: bool = False, trace: bool = False, quiet: bool = False, nb_threads: int = 5 + ) -> None: """ Initialise the ThreadedScanning class :param scanapi: SCANOSS API to send scan requests to @@ -158,8 +162,9 @@ def run(self, wait: bool = True) -> bool: """ qsize = self.inputs.qsize() if qsize < self.nb_threads: - self.print_debug(f'Input queue ({qsize}) smaller than requested threads: {self.nb_threads}. ' - f'Reducing to queue size.') + self.print_debug( + f'Input queue ({qsize}) smaller than requested threads: {self.nb_threads}. Reducing to queue size.' + ) self.nb_threads = qsize else: self.print_debug(f'Starting {self.nb_threads} threads to process {qsize} requests...') @@ -171,7 +176,7 @@ def run(self, wait: bool = True) -> bool: except Exception as e: self.print_stderr(f'ERROR: Problem running threaded scanning: {e}') self._errors = True - if wait: # Wait for all inputs to complete + if wait: # Wait for all inputs to complete self.complete() return False if self._errors else True @@ -180,7 +185,7 @@ def complete(self) -> bool: Wait for input queue to complete processing and complete the worker threads """ self.inputs.join() - self._stop_event.set() # Tell the worker threads to stop + self._stop_event.set() # Tell the worker threads to stop try: for t in self._threads: # Complete the threads t.join(timeout=5) @@ -199,7 +204,7 @@ def worker_post(self) -> None: api_error = False while not self._stop_event.is_set(): wfp = None - if not self.inputs.empty(): # Only try to get a message if there is one on the queue + if not self.inputs.empty(): # Only try to get a message if there is one on the queue try: wfp = self.inputs.get(timeout=5) if api_error: # API error encountered, so stop processing anymore requests @@ -228,6 +233,7 @@ def worker_post(self) -> None: time.sleep(1) # Sleep while waiting for the queue depth to build up self.print_trace(f'Thread complete ({current_thread}).') + # # End of ThreadedScanning Class # diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py index e64d7d7a..ab28327c 100644 --- a/src/scanoss/utils/file.py +++ b/src/scanoss/utils/file.py @@ -52,24 +52,24 @@ def validate_json_file(json_file_path: str) -> JsonValidation: Tuple[bool, str]: A tuple containing a boolean indicating if the file is valid and a message """ if not json_file_path: - return JsonValidation(is_valid=False, error="No JSON file specified") + return JsonValidation(is_valid=False, error='No JSON file specified') if not os.path.isfile(json_file_path): return JsonValidation( is_valid=False, - error=f"File not found: {json_file_path}", + error=f'File not found: {json_file_path}', error_code=JSON_ERROR_FILE_NOT_FOUND, ) try: if os.stat(json_file_path).st_size == 0: return JsonValidation( is_valid=False, - error=f"File is empty: {json_file_path}", + error=f'File is empty: {json_file_path}', error_code=JSON_ERROR_FILE_EMPTY, ) except OSError as e: return JsonValidation( is_valid=False, - error=f"Problem checking file size: {json_file_path}: {e}", + error=f'Problem checking file size: {json_file_path}: {e}', error_code=JSON_ERROR_FILE_SIZE, ) try: diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 4026ada6..02953d73 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -1,32 +1,33 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. - Winnowing Algorithm implementation for SCANOSS. + Winnowing Algorithm implementation for SCANOSS. - This module implements an adaptation of the original winnowing algorithm by S. Schleimer, D. S. Wilkerson and - A. Aiken as described in their seminal article which can be found here: - https://theory.stanford.edu/~aiken/publications/papers/sigmod03.pdf + This module implements an adaptation of the original winnowing algorithm by S. Schleimer, D. S. Wilkerson and + A. Aiken as described in their seminal article which can be found here: + https://theory.stanford.edu/~aiken/publications/papers/sigmod03.pdf """ + import hashlib import pathlib import platform @@ -55,11 +56,56 @@ MIN_FILE_SIZE = 256 SKIP_SNIPPET_EXT = { # File extensions to ignore snippets for - ".exe", ".zip", ".tar", ".tgz", ".gz", ".7z", ".rar", ".jar", ".war", ".ear", ".class", ".pyc", - ".o", ".a", ".so", ".obj", ".dll", ".lib", ".out", ".app", ".bin", - ".lst", ".dat", ".json", ".htm", ".html", ".xml", ".md", ".txt", - ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".odt", ".ods", ".odp", ".pages", ".key", ".numbers", - ".pdf", ".min.js", ".mf", ".sum", ".woff", ".woff2", '.xsd', ".pom", ".whl", + '.exe', + '.zip', + '.tar', + '.tgz', + '.gz', + '.7z', + '.rar', + '.jar', + '.war', + '.ear', + '.class', + '.pyc', + '.o', + '.a', + '.so', + '.obj', + '.dll', + '.lib', + '.out', + '.app', + '.bin', + '.lst', + '.dat', + '.json', + '.htm', + '.html', + '.xml', + '.md', + '.txt', + '.doc', + '.docx', + '.xls', + '.xlsx', + '.ppt', + '.pptx', + '.odt', + '.ods', + '.odp', + '.pages', + '.key', + '.numbers', + '.pdf', + '.min.js', + '.mf', + '.sum', + '.woff', + '.woff2', + '.xsd', + '.pom', + '.whl', } CRC8_MAXIM_DOW_TABLE_SIZE = 0x100 @@ -111,11 +157,21 @@ class Winnowing(ScanossBase): a list of WFP fingerprints with their corresponding line numbers. """ - def __init__(self, size_limit: bool = False, debug: bool = False, trace: bool = False, quiet: bool = False, - skip_snippets: bool = False, post_size: int = 32, all_extensions: bool = False, - obfuscate: bool = False, hpsm: bool = False, - strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None - ): + def __init__( + self, + size_limit: bool = False, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + skip_snippets: bool = False, + post_size: int = 32, + all_extensions: bool = False, + obfuscate: bool = False, + hpsm: bool = False, + strip_hpsm_ids=None, + strip_snippet_ids=None, + skip_md5_ids=None, + ): """ Instantiate Winnowing class Parameters @@ -190,12 +246,16 @@ def __skip_snippets(self, file: str, src: str) -> bool: if src_len == 0 or src_len <= MIN_FILE_SIZE: # Ignore empty or files that are too small self.print_trace(f'Skipping snippets as the file is too small: {file} - {src_len}') return True - prefix = src[0:(MIN_FILE_SIZE - 1)].lower().strip() - if len(prefix) > 0 and (prefix[0] == "{" or prefix[0] == "["): # Ignore json + prefix = src[0 : (MIN_FILE_SIZE - 1)].lower().strip() + if len(prefix) > 0 and (prefix[0] == '{' or prefix[0] == '['): # Ignore json self.print_trace(f'Skipping snippets as the file appears to be JSON: {file}') return True - if prefix.startswith(" str: elif hpsm_id_len % 2 == 1: hpsm_id_len = hpsm_id_len + 1 - to_remove = hpsm[hpsm_id_index:hpsm_id_index + hpsm_id_len] + to_remove = hpsm[hpsm_id_index : hpsm_id_index + hpsm_id_len] self.print_debug(f'HPSM ID {to_remove} to replace') # Calculate the XOR of each byte to produce the correct ignore sequence. replacement = ''.join( - [format(int(to_remove[i:i + 2], 16) ^ 0xFF, '02x') for i in range(0, len(to_remove), 2)]) + [format(int(to_remove[i : i + 2], 16) ^ 0xFF, '02x') for i in range(0, len(to_remove), 2)] + ) self.print_debug(f'HPSM ID replacement {replacement}') # Overwrite HPSM bytes to be removed. @@ -309,7 +370,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # Print file line content_length = len(contents) original_filename = file - + if platform.system() == 'Windows': original_filename = file.replace('\\', '/') wfp_filename = repr(original_filename).strip("'") # return a utf-8 compatible version of the filename @@ -361,14 +422,16 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: crc_hex = '{:08x}'.format(crc) if last_line != line: if output != '': - if self.size_limit and \ - (len(wfp.encode("utf-8")) + len( - output.encode("utf-8"))) > self.max_post_size: + if ( + self.size_limit + and (len(wfp.encode('utf-8')) + len(output.encode('utf-8'))) + > self.max_post_size + ): self.print_debug(f'Truncating WFP ({self.max_post_size} limit) for: {file}') output = '' break # Stop collecting snippets as it's over 64k wfp += output + '\n' - output = "%d=%s" % (line, crc_hex) + output = '%d=%s' % (line, crc_hex) else: output += ',' + crc_hex @@ -379,7 +442,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # Shift gram gram = gram[1:] if output != '': - if not self.size_limit or (len(wfp.encode("utf-8")) + len(output.encode("utf-8"))) < self.max_post_size: + if not self.size_limit or (len(wfp.encode('utf-8')) + len(output.encode('utf-8'))) < self.max_post_size: wfp += output + '\n' else: self.print_debug(f'Warning: skipping output in WFP for {file} - "{output}"') @@ -403,13 +466,13 @@ def calc_hpsm(self, content): last_line = 0 for i, byte in enumerate(content): c = byte - if c == ASCII_LF: # When there is a new line - if len(list_normalized): + if c == ASCII_LF: # When there is a new line + if len(list_normalized): crc_lines.append(self.crc8_buffer(list_normalized)) list_normalized = [] - elif last_line+1 == i: + elif last_line + 1 == i: crc_lines.append(0xFF) - elif i-last_line > 1: + elif i - last_line > 1: crc_lines.append(0x00) last_line = i else: @@ -470,6 +533,7 @@ def crc8_buffer(self, buffer): crc ^= CRC8_MAXIM_DOW_FINAL # Bitwise OR (XOR) of crc in Maxim Dow Final return crc + # # End of Winnowing Class # diff --git a/tests/test_csv_output.py b/tests/test_csv_output.py index 73e7bcd8..499e38e4 100644 --- a/tests/test_csv_output.py +++ b/tests/test_csv_output.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import unittest from scanoss.winnowing import Winnowing @@ -30,11 +31,12 @@ class MyTestCase(unittest.TestCase): """ Exercise the Winnowing class """ + def test_csv_output(self): winnowing = Winnowing(debug=True) - filename = "test-file.c" - contents = "c code contents" - content_types = bytes(contents, encoding="raw_unicode_escape") + filename = 'test-file.c' + contents = 'c code contents' + content_types = bytes(contents, encoding='raw_unicode_escape') wfp = winnowing.wfp_for_contents(filename, False, content_types) print(f'WFP for {filename}: {wfp}') self.assertIsNotNone(wfp) diff --git a/tests/test_file_filters.py b/tests/test_file_filters.py index fef91874..8e5dcf3c 100644 --- a/tests/test_file_filters.py +++ b/tests/test_file_filters.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import os import shutil import tempfile @@ -146,20 +147,26 @@ def test_size_limits(self): else: f.write('a' * 100) - file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True, operation_type='scanning') + file_filters = FileFilters( + debug=True, scanoss_settings=settings, hidden_files_folders=True, operation_type='scanning' + ) # For scanning, only *.py files have size limits filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), ['file1.js', 'file2.py']) - file_filters = FileFilters(debug=True, scanoss_settings=settings, hidden_files_folders=True, operation_type='fingerprinting') + file_filters = FileFilters( + debug=True, scanoss_settings=settings, hidden_files_folders=True, operation_type='fingerprinting' + ) # For fingerprinting, all files have size limits filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), ['file2.py']) def test_all_extensions_flag(self): - file_filters = FileFilters(debug=True, all_extensions=True, hidden_files_folders=True, operation_type='scanning') + file_filters = FileFilters( + debug=True, all_extensions=True, hidden_files_folders=True, operation_type='scanning' + ) files = [ 'file1.js', 'file2.css', # Would normally be skipped @@ -210,7 +217,7 @@ def test_hidden_files_and_folders_enabled(self): '.hidden_dir/.nested_hidden_file.js', 'visible_dir/.hidden_file.go', '.git/config', - '.hidden_dir/nested_dir/.hidden_nested_file.py' + '.hidden_dir/nested_dir/.hidden_nested_file.py', ] self.create_files(files) @@ -219,7 +226,7 @@ def test_hidden_files_and_folders_enabled(self): '.hidden_dir/visible_file.py', '.hidden_dir/.nested_hidden_file.js', 'visible_dir/.hidden_file.go', - '.hidden_dir/nested_dir/.hidden_nested_file.py' + '.hidden_dir/nested_dir/.hidden_nested_file.py', ] filtered_files = self.file_filters.get_filtered_files_from_folder(self.test_dir) @@ -233,13 +240,11 @@ def test_hidden_files_and_folders_disabled(self): '.hidden_dir/.nested_hidden_file.js', 'visible_dir/.hidden_file.go', 'visible_file.py', - '.git/config' + '.git/config', ] self.create_files(files) - expected_files = [ - 'visible_file.py' - ] + expected_files = ['visible_file.py'] filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) @@ -253,7 +258,7 @@ def test_all_extensions_mode(self): '.hidden_file.dat', 'dir1/file4.bmp', 'dir1/.hidden/file5.class', - 'file6.py' + 'file6.py', ] self.create_files(files) @@ -264,7 +269,7 @@ def test_all_extensions_mode(self): '.hidden_file.dat', 'dir1/file4.bmp', 'dir1/.hidden/file5.class', - 'file6.py' + 'file6.py', ] filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) @@ -278,7 +283,7 @@ def test_all_folders_mode(self): 'eggs/module.py', 'wheels/util.py', 'normal_dir/file.py', - '.git/config.py' + '.git/config.py', ] self.create_files(files) @@ -288,20 +293,22 @@ def test_all_folders_mode(self): 'eggs/module.py', 'wheels/util.py', 'normal_dir/file.py', - '.git/config.py' + '.git/config.py', ] filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) self.assertEqual(sorted(filtered_files), sorted(expected_files)) def test_combined_all_modes(self): - file_filters = FileFilters(debug=True, all_extensions=True, all_folders=True, hidden_files_folders=True, operation_type='scanning') + file_filters = FileFilters( + debug=True, all_extensions=True, all_folders=True, hidden_files_folders=True, operation_type='scanning' + ) files = [ '.hidden_dir/file1.css', '__pycache__/cache.dat', 'venv/.hidden_file.class', 'normal_dir/file.py', - '.config/settings.bmp' + '.config/settings.bmp', ] self.create_files(files) @@ -310,7 +317,7 @@ def test_combined_all_modes(self): '__pycache__/cache.dat', 'venv/.hidden_file.class', 'normal_dir/file.py', - '.config/settings.bmp' + '.config/settings.bmp', ] filtered_files = file_filters.get_filtered_files_from_folder(self.test_dir) diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index ef713bd7..d39e0cdf 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import json import os import re @@ -31,15 +32,14 @@ class MyTestCase(unittest.TestCase): - - """ Inspect for copyleft licenses """ + def test_copyleft_policy(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" - input_file_name = os.path.join(script_dir,'data', file_name) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json') copyleft.run() self.assertEqual(True, True) @@ -47,21 +47,22 @@ def test_copyleft_policy(self): """ Inspect for copyleft licenses empty path """ + def test_copyleft_policy_empty_path(self): copyleft = Copyleft(filepath='', format_type='json') success, results = copyleft.run() - self.assertTrue(success,2) - + self.assertTrue(success, 2) """ Inspect for empty copyleft licenses """ + def test_empty_copyleft_policy(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result-no-copyleft.json" - input_file_name = os.path.join(script_dir,'data', file_name) + file_name = 'result-no-copyleft.json' + input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json') - status,results = copyleft.run() + status, results = copyleft.run() details = json.loads(results['details']) self.assertEqual(status, 1) self.assertEqual(details, {}) @@ -70,9 +71,10 @@ def test_empty_copyleft_policy(self): """ Inspect for copyleft licenses include """ + def test_copyleft_policy_include(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', include='MIT') status, results = copyleft.run() @@ -84,18 +86,19 @@ def test_copyleft_policy_include(self): has_mit_license = True break - self.assertEqual(status,0) + self.assertEqual(status, 0) self.assertEqual(has_mit_license, True) """ Inspect for copyleft licenses exclude """ + def test_copyleft_policy_exclude(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', exclude='GPL-2.0-only') - status,results = copyleft.run() + status, results = copyleft.run() details = json.loads(results['details']) self.assertEqual(details, {}) self.assertEqual(status, 1) @@ -103,47 +106,53 @@ def test_copyleft_policy_exclude(self): """ Inspect for copyleft licenses explicit """ + def test_copyleft_policy_explicit(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='MIT') status, results = copyleft.run() details = json.loads(results['details']) self.assertEqual(len(details['components']), 3) - self.assertEqual(status,0) + self.assertEqual(status, 0) """ Inspect for copyleft licenses empty explicit licenses (should set the default ones) """ + def test_copyleft_policy_empty_explicit(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='') status, results = copyleft.run() details = json.loads(results['details']) self.assertEqual(len(details['components']), 5) - self.assertEqual(status,0) - + self.assertEqual(status, 0) """ Export copyleft licenses in Markdown """ + def test_copyleft_policy_markdown(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='md', explicit='MIT') status, results = copyleft.run() - expected_detail_output = ('### Copyleft licenses \n | Component | Version | License | URL | Copyleft |\n' - ' | - | :-: | - | - | :-: |\n' - '| pkg:github/scanoss/engine | 4.0.4 | MIT | https://spdx.org/licenses/MIT.html | YES | \n' - ' | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | https://spdx.org/licenses/MIT.html | YES |\n' - '| pkg:npm/%40emotion/react | 11.13.3 | MIT | https://spdx.org/licenses/MIT.html | YES | \n') + expected_detail_output = ( + '### Copyleft licenses \n | Component | Version | License | URL | Copyleft |\n' + ' | - | :-: | - | - | :-: |\n' + '| pkg:github/scanoss/engine | 4.0.4 | MIT | https://spdx.org/licenses/MIT.html | YES | \n' + ' | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | https://spdx.org/licenses/MIT.html | YES |\n' + '| pkg:npm/%40emotion/react | 11.13.3 | MIT | https://spdx.org/licenses/MIT.html | YES | \n' + ) expected_summary_output = '3 component(s) with copyleft licenses were found.\n' - self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', results['details']), - re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output)) + self.assertEqual( + re.sub(r'\s|\\(?!`)|\\(?=`)', '', results['details']), + re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output), + ) self.assertEqual(results['summary'], expected_summary_output) self.assertEqual(status, 0) @@ -152,19 +161,20 @@ def test_copyleft_policy_markdown(self): """ Inspect for undeclared components empty path """ + def test_copyleft_policy_empty_path(self): copyleft = Copyleft(filepath='', format_type='json') success, results = copyleft.run() - self.assertTrue(success,2) - + self.assertTrue(success, 2) """ Inspect for undeclared components """ + def test_undeclared_policy(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" - input_file_name = os.path.join(script_dir,'data', file_name) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='json', sbom_format='legacy') status, results = undeclared.run() details = json.loads(results['details']) @@ -190,22 +200,24 @@ def test_undeclared_policy(self): }``` """ self.assertEqual(len(details['components']), 5) - self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', - '', expected_summary_output)) + self.assertEqual( + re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) + ) self.assertEqual(status, 0) """ Undeclared component markdown output """ + def test_undeclared_policy_markdown(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md', sbom_format='legacy') status, results = undeclared.run() details = results['details'] summary = results['summary'] - expected_details_output = """ ### Undeclared components + expected_details_output = """ ### Undeclared components | Component | Version | License | | - | - | - | | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | @@ -237,17 +249,20 @@ def test_undeclared_policy_markdown(self): print(summary) self.assertEqual(status, 0) - self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', - '', expected_details_output)) - self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), - re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) + self.assertEqual( + re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) + ) + self.assertEqual( + re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) + ) """ Undeclared component markdown scanoss summary output """ + def test_undeclared_policy_markdown_scanoss_summary(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md') status, results = undeclared.run() @@ -288,17 +303,20 @@ def test_undeclared_policy_markdown_scanoss_summary(self): print(summary) self.assertEqual(status, 0) - self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', - '', expected_details_output)) - self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), - re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) + self.assertEqual( + re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) + ) + self.assertEqual( + re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) + ) """ Undeclared component sbom summary output """ + def test_undeclared_policy_scanoss_summary(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name) status, results = undeclared.run() @@ -329,12 +347,13 @@ def test_undeclared_policy_scanoss_summary(self): ```""" self.assertEqual(status, 0) self.assertEqual(len(details['components']), 5) - self.assertEqual(re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), - re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output)) + self.assertEqual( + re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) + ) def test_undeclared_policy_jira_markdown_output(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='jira_md') status, results = undeclared.run() @@ -376,7 +395,7 @@ def test_undeclared_policy_jira_markdown_output(self): def test_copyleft_policy_jira_markdown_output(self): script_dir = os.path.dirname(os.path.abspath(__file__)) - file_name = "result.json" + file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='jira_md') status, results = copyleft.run() @@ -392,6 +411,5 @@ def test_copyleft_policy_jira_markdown_output(self): self.assertEqual(expected_details_output, details) - if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_winnowing.py b/tests/test_winnowing.py index 52e05e43..59c6f0a3 100644 --- a/tests/test_winnowing.py +++ b/tests/test_winnowing.py @@ -1,26 +1,27 @@ """ - SPDX-License-Identifier: MIT +SPDX-License-Identifier: MIT - Copyright (c) 2021, SCANOSS + Copyright (c) 2021, SCANOSS - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import unittest from scanoss.winnowing import Winnowing @@ -30,11 +31,12 @@ class MyTestCase(unittest.TestCase): """ Exercise the Winnowing class """ + def test_winnowing(self): winnowing = Winnowing(debug=True) - filename = "test-file.c" - contents = "c code contents" - content_types = bytes(contents, encoding="raw_unicode_escape") + filename = 'test-file.c' + contents = 'c code contents' + content_types = bytes(contents, encoding='raw_unicode_escape') wfp = winnowing.wfp_for_contents(filename, False, content_types) print(f'WFP for {filename}: {wfp}') self.assertIsNotNone(wfp) @@ -45,18 +47,18 @@ def test_winnowing(self): def test_snippet_skip(self): winnowing = Winnowing(debug=True) - filename = "test-file.jar" - contents = "jar file contents" - content_types = bytes(contents, encoding="raw_unicode_escape") + filename = 'test-file.jar' + contents = 'jar file contents' + content_types = bytes(contents, encoding='raw_unicode_escape') wfp = winnowing.wfp_for_contents(filename, False, content_types) print(f'WFP for {filename}: {wfp}') self.assertIsNotNone(wfp) - + def test_snippet_strip(self): - winnowing = Winnowing(debug=True, hpsm=True, - strip_snippet_ids=['d5e54c33,b03faabe'], - strip_hpsm_ids=['0d2fffaffc62d18']) - filename = "test-file.py" + winnowing = Winnowing( + debug=True, hpsm=True, strip_snippet_ids=['d5e54c33,b03faabe'], strip_hpsm_ids=['0d2fffaffc62d18'] + ) + filename = 'test-file.py' with open(__file__, 'rb') as f: contents = f.read() print('--- Test snippet and HPSM strip ---') @@ -67,8 +69,8 @@ def test_snippet_strip(self): found = wfp.index('d5e54c33,b03faabe') except ValueError: found = -1 - self.assertEqual(found, -1) - + self.assertEqual(found, -1) + try: found = wfp.index('0d2fffaffc62d18') except ValueError: @@ -76,6 +78,5 @@ def test_snippet_strip(self): self.assertEqual(found, -1) - if __name__ == '__main__': unittest.main() diff --git a/version.py b/version.py index 51d764ab..022888ab 100755 --- a/version.py +++ b/version.py @@ -1,27 +1,28 @@ #!/usr/bin/env python3 """ - SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. +SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. """ + import os import codecs @@ -48,11 +49,11 @@ def get_version(rel_path): delim = '"' if '"' in line else "'" return line.split(delim)[1] else: - raise RuntimeError("Unable to find version string.") + raise RuntimeError('Unable to find version string.') """ Load __init__.py from the scanoss package and print the version to stdout """ -if __name__ == "__main__": +if __name__ == '__main__': print(get_version('src/scanoss/__init__.py')) From 68bee23c1e95904a207a5c3ea5e6bc511f3bebb0 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 4 Feb 2025 12:03:19 +0100 Subject: [PATCH 273/489] chore: exclude test files from lint --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 29a06a43..8cde7b9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ select = ["E", "F", "I", "PL"] line-length = 120 # Assume Python 3.7+ target-version = "py37" +exclude = ["tests/*", "test_*.py"] [tool.ruff.format] quote-style = "single" From 261a39f4d2245048d9f89071055214cac0dc4ad0 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 10 Feb 2025 12:10:27 +0100 Subject: [PATCH 274/489] chore: exclude test files from lint --- src/scanoss/cli.py | 77 +++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index be35b2b4..aa4e0ba3 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -24,25 +24,25 @@ import argparse import os -from pathlib import Path import sys +from pathlib import Path + import pypac +from . import __version__ +from .components import Components +from .csvoutput import CsvOutput +from .cyclonedx import CycloneDx +from .filecount import FileCount from .inspection.copyleft import Copyleft from .inspection.undeclared_component import UndeclaredComponent -from .threadeddependencies import SCOPE -from .scanoss_settings import ScanossSettings, ScanossSettingsError +from .results import Results from .scancodedeps import ScancodeDeps -from .scanner import Scanner +from .scanner import FAST_WINNOWING, Scanner +from .scanoss_settings import ScanossSettings, ScanossSettingsError from .scantype import ScanType -from .filecount import FileCount -from .cyclonedx import CycloneDx from .spdxlite import SpdxLite -from .csvoutput import CsvOutput -from .components import Components -from . import __version__ -from .scanner import FAST_WINNOWING -from .results import Results +from .threadeddependencies import SCOPE from .utils.file import validate_json_file @@ -284,11 +284,13 @@ def setup_args() -> None: c_semgrep.set_defaults(func=comp_semgrep) # Component Sub-command: component provenance - c_provenance = comp_sub.add_parser('provenance', aliases=['prov', 'prv'], - description=f'Show Provenance findings: {__version__}', - help='Retrieve provenance for the given components') - c_provenance.set_defaults(func=comp_provenance) - + c_provenance = comp_sub.add_parser( + 'provenance', + aliases=['prov', 'prv'], + description=f'Show Provenance findings: {__version__}', + help='Retrieve provenance for the given components', + ) + c_provenance.set_defaults(func=c_provenance) # Component Sub-command: component search c_search = comp_sub.add_parser( @@ -589,18 +591,17 @@ def setup_args() -> None: if not args.subparser: parser.print_help() # No sub command subcommand, print general help exit(1) - else: - if ( - args.subparser == 'utils' - or args.subparser == 'ut' - or args.subparser == 'component' - or args.subparser == 'comp' - or args.subparser == 'inspect' - or args.subparser == 'insp' - or args.subparser == 'ins' - ) and not args.subparsercmd: - parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed - exit(1) + elif ( + args.subparser == 'utils' + or args.subparser == 'ut' + or args.subparser == 'component' + or args.subparser == 'comp' + or args.subparser == 'inspect' + or args.subparser == 'insp' + or args.subparser == 'ins' + ) and not args.subparsercmd: + parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed + exit(1) args.func(parser, args) # Execute the function associated with the sub-command @@ -671,7 +672,7 @@ def wfp(parser, args): parser.parse_args([args.subparser, '-h']) exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: - print_stderr(f'Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') + print_stderr('Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') scan_output: str = None if args.output: scan_output = args.output @@ -746,11 +747,11 @@ def get_scan_options(args): if args.debug: if ScanType.SCAN_FILES.value & scan_options: - print_stderr(f'Scan Files') + print_stderr('Scan Files') if ScanType.SCAN_SNIPPETS.value & scan_options: - print_stderr(f'Scan Snippets') + print_stderr('Scan Snippets') if ScanType.SCAN_DEPENDENCIES.value & scan_options: - print_stderr(f'Scan Dependencies') + print_stderr('Scan Dependencies') if scan_options <= 0: print_stderr(f'Error: No valid scan options configured: {scan_options}') exit(1) @@ -907,7 +908,7 @@ def scan(parser, args): print_stderr(f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})') exit(1) if scanner.is_dependency_scan() and not args.dep: - print_stderr(f'Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)') + print_stderr('Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)') exit(1) scanner.scan_wfp_with_options(args.wfp, args.dep) elif args.stdin: @@ -947,7 +948,7 @@ def scan(parser, args): elif args.dep: if not args.dependencies_only: print_stderr( - f'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.' + 'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.' ) exit(1) if not scanner.scan_folder_with_options( @@ -1005,17 +1006,17 @@ def convert(parser, args): success = False if args.format == 'cyclonedx': if not args.quiet: - print_stderr(f'Producing CycloneDX report...') + print_stderr('Producing CycloneDX report...') cdx = CycloneDx(debug=args.debug, output_file=args.output) success = cdx.produce_from_file(args.input) elif args.format == 'spdxlite': if not args.quiet: - print_stderr(f'Producing SPDX Lite report...') + print_stderr('Producing SPDX Lite report...') spdxlite = SpdxLite(debug=args.debug, output_file=args.output) success = spdxlite.produce_from_file(args.input) elif args.format == 'csv': if not args.quiet: - print_stderr(f'Producing CSV report...') + print_stderr('Producing CSV report...') csvo = CsvOutput(debug=args.debug, output_file=args.output) success = csvo.produce_from_file(args.input) else: @@ -1171,7 +1172,7 @@ def utils_pac_proxy(_, args): from pypac.resolver import ProxyResolver if not args.pac: - print_stderr(f'Error: No pac file option specified.') + print_stderr('Error: No pac file option specified.') exit(1) pac_file = get_pac_file(args.pac) if pac_file is None: From 38bf356b15684ed45dbf9379f96317b05bffb533 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Wed, 19 Feb 2025 10:40:22 -0300 Subject: [PATCH 275/489] feat: Enhance SPDXLite export format for Telco compliance * feat:SP-2115 Enhance SPDXLite output for Telco compliance * chore:SP-2120 Adds SPDXLite unit test * chore: Upgrades scanoss-py version to 1.20.1 --- CHANGELOG.md | 7 +- src/scanoss/__init__.py | 2 +- src/scanoss/spdxlite.py | 391 ++++++++++++++++++++++++---------------- tests/test_spdxlite.py | 70 +++++++ 4 files changed, 317 insertions(+), 153 deletions(-) create mode 100644 tests/test_spdxlite.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cd450b0..53db4de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.1] - 2025-02-18 +### Added +- Enhanced SPDX Lite report to achieve Telco compliance + ## [1.20.0] - 2025-02-02 ### Added - Added support for component provenance reporting @@ -456,4 +460,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.19.3]: https://github.com/scanoss/scanoss.py/compare/v1.19.2...v1.19.3 [1.19.4]: https://github.com/scanoss/scanoss.py/compare/v1.19.3...v1.19.4 [1.19.5]: https://github.com/scanoss/scanoss.py/compare/v1.19.4...v1.19.5 -[1.20.0]: https://github.com/scanoss/scanoss.py/compare/v1.19.5...v1.20.0 \ No newline at end of file +[1.20.0]: https://github.com/scanoss/scanoss.py/compare/v1.19.5...v1.20.0 +[1.20.1]: https://github.com/scanoss/scanoss.py/compare/v1.20.0...v1.20.1 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 3a0140c9..733f4109 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.19.6' +__version__ = '1.20.1' diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index b8bf5c6b..b24b178d 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -22,13 +22,14 @@ THE SOFTWARE. """ -import json -import os.path -import sys -import hashlib import datetime import getpass +import hashlib +import json +import os.path import re +import sys + import importlib_resources from . import __version__ @@ -72,75 +73,103 @@ def parse(self, data: json): if not data: self.print_stderr('ERROR: No JSON data provided to parse.') return None - self.print_debug(f'Processing raw results into summary format...') + + self.print_debug('Processing raw results into summary format...') + return self._process_files(data) + + def _process_files(self, data: json) -> dict: + """Process each file in the data and build summary.""" summary = {} - for f in data: - file_details = data.get(f) - # print(f'File: {f}: {file_details}\n') - for d in file_details: - id_details = d.get('id') - if not id_details or id_details == 'none': # Ignore files with no ids - continue - purl = None - if id_details == 'dependency': # Process dependency data - dependencies = d.get('dependencies') - if not dependencies: - self.print_stderr(f'Warning: No Dependencies found for {f}: {file_details}') - continue - for deps in dependencies: - # print(f'File: {f} Deps: {deps}') - purl = deps.get('purl') - if not purl: - self.print_stderr(f'Warning: No PURL found for {f}: {deps}') - continue - if summary.get(purl): - self.print_debug(f'Component {purl} already stored: {summary.get(purl)}') - continue - fd = {} - for field in ['component', 'version', 'url']: - fd[field] = deps.get(field, '') - licenses = deps.get('licenses') - fdl = [] - if licenses: - dc = [] - for lic in licenses: - name = lic.get('name') - if name not in dc: # Only save the license name once - fdl.append({'id': name}) - dc.append(name) - fd['licenses'] = fdl - summary[purl] = fd - else: # Normal file id type - purls = d.get('purl') - if not purls: - self.print_stderr(f'Purl block missing for {f}: {file_details}') - continue - for p in purls: - self.print_debug(f'Purl: {p}') - purl = p - break - if not purl: - self.print_stderr(f'Warning: No PURL found for {f}: {file_details}') - continue - if summary.get(purl): - self.print_debug(f'Component {purl} already stored: {summary.get(purl)}') - continue - fd = {} - for field in ['id', 'vendor', 'component', 'version', 'latest', 'url']: - fd[field] = d.get(field) - licenses = d.get('licenses') - fdl = [] - if licenses: - dc = [] - for lic in licenses: - name = lic.get('name') - if name not in dc: # Only save the license name once - fdl.append({'id': name}) - dc.append(name) - fd['licenses'] = fdl - summary[purl] = fd + for file_path in data: + file_details = data.get(file_path) + self._process_file_entries(file_path, file_details, summary) return summary + def _process_file_entries(self, file_path: str, file_details: list, summary: dict): + """Process entries for a single file.""" + for entry in file_details: + id_details = entry.get('id') + if not id_details or id_details == 'none': + continue + + if id_details == 'dependency': + self._process_dependency_entry(file_path, entry, summary) + else: + self._process_normal_entry(file_path, entry, summary) + + def _process_dependency_entry(self, file_path: str, entry: dict, summary: dict): + """Process a dependency type entry.""" + dependencies = entry.get('dependencies') + if not dependencies: + self.print_stderr(f'Warning: No Dependencies found for {file_path}') + return + + for dep in dependencies: + purl = dep.get('purl') + if not self._is_valid_purl(file_path, dep, purl, summary): + continue + + summary[purl] = self._create_dependency_summary(dep) + + def _process_normal_entry(self, file_path: str, entry: dict, summary: dict): + """Process a normal file type entry.""" + purls = entry.get('purl') + if not purls: + self.print_stderr(f'Purl block missing for {file_path}') + return + + purl = purls[0] if purls else None + if not self._is_valid_purl(file_path, entry, purl, summary): + return + + summary[purl] = self._create_normal_summary(entry) + + def _is_valid_purl(self, file_path: str, entry: dict, purl: str, summary: dict) -> bool: + """Check if PURL is valid and not already processed.""" + if not purl: + self.print_stderr(f'Warning: No PURL found for {file_path}: {entry}') + return False + + if summary.get(purl): + self.print_debug(f'Component {purl} already stored: {summary.get(purl)}') + return False + + return True + + def _create_dependency_summary(self, dep: dict) -> dict: + """Create summary for dependency entry.""" + summary = {} + for field in ['component', 'version', 'url']: + summary[field] = dep.get(field, '') + summary['licenses'] = self._process_licenses(dep.get('licenses')) + return summary + + def _create_normal_summary(self, entry: dict) -> dict: + """Create summary for normal file entry.""" + summary = {} + fields = ['id', 'vendor', 'component', 'version', 'latest', + 'url', 'url_hash', 'download_url'] + for field in fields: + summary[field] = entry.get(field) + summary['licenses'] = self._process_licenses(entry.get('licenses')) + return summary + + def _process_licenses(self, licenses: list) -> list: + """Process license information and remove duplicates.""" + if not licenses: + return [] + + processed_licenses = [] + seen_names = set() + + for license_info in licenses: + name = license_info.get('name') + if name and name not in seen_names: + processed_licenses.append({'id': name}) + seen_names.add(name) + + return processed_licenses + def produce_from_file(self, json_file: str, output_file: str = None) -> bool: """ Parse plain/raw input JSON file and produce SPDX Lite output @@ -169,103 +198,163 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: if not raw_data: self.print_stderr('ERROR: No SPDX data returned for the JSON string provided.') return False + self.load_license_data() - # Using this SPDX version as the spec - # https://github.com/spdx/spdx-spec/blob/development/v2.2.2/examples/SPDXJSONExample-v2.2.spdx.json - # Validate using: - # pip3 install jsonschema - # jsonschema -i spdxlite.json <(curl https://raw.githubusercontent.com/spdx/spdx-spec/v2.2/schemas/spdx-schema.json) - # Validation can also be done online here: https://tools.spdx.org/app/validate/ - now = datetime.datetime.utcnow() # TODO replace with recommended format + spdx_document = self._create_base_document(raw_data) + self._process_packages(raw_data, spdx_document) + return self._write_output(spdx_document, output_file) + + def _create_base_document(self, raw_data: dict) -> dict: + """Create the base SPDX document structure.""" + now = datetime.datetime.utcnow() md5hex = hashlib.md5(f'{raw_data}-{now}'.encode('utf-8')).hexdigest() - data = { + + return { 'spdxVersion': 'SPDX-2.2', 'dataLicense': 'CC0-1.0', - 'SPDXID': f'SPDXRef-DOCUMENT', + 'SPDXID': 'SPDXRef-DOCUMENT', 'name': 'SCANOSS-SBOM', - 'creationInfo': { - 'created': now.strftime('%Y-%m-%dT%H:%M:%SZ'), - 'creators': [f'Tool: SCANOSS-PY: {__version__}', f'Person: {getpass.getuser()}'], - }, + 'creationInfo': self._create_creation_info(now), 'documentNamespace': f'https://spdx.org/spdxdocs/scanoss-py-{__version__}-{md5hex}', 'documentDescribes': [], 'hasExtractedLicensingInfos': [], 'packages': [], } - lic_refs = set() # Hash Set of non-SPDX license references - for purl in raw_data: - comp = raw_data.get(purl) - licenses = comp.get('licenses') - lic_text = 'NOASSERTION' - if licenses: - lic_set = set() - for lic in licenses: - lc_id = lic.get('id') - if lc_id: - spdx_id = self.get_spdx_license_id(lc_id) - if not spdx_id: - if not lc_id.startswith('LicenseRef'): - lc_id = f'LicenseRef-{lc_id}' # Make sure it has a license ref in its name - lic_refs.add(lc_id) # save non-SPDX license for later reference - lic_set.add(spdx_id if spdx_id else lc_id) - if len(lic_set) > 0: - lic_text = ' AND '.join(lic_set) - if len(lic_set) > 1: - lic_text = f'({lic_text})' # wrap the names in () if there is more than one - comp_name = comp.get('component') - comp_ver = comp.get('version') - purl_ver = f'{purl}@{comp_ver}' - vendor = comp.get('vendor', 'NOASSERTION') - supplier = f'Organization: {vendor}' if vendor != 'NOASSERTION' else vendor - purl_hash = hashlib.md5(f'{purl_ver}'.encode('utf-8')).hexdigest() - purl_spdx = f'SPDXRef-{purl_hash}' - data['documentDescribes'].append(purl_spdx) - data['packages'].append( + + def _create_creation_info(self, timestamp: datetime.datetime) -> dict: + """Create the creation info section.""" + return { + 'created': timestamp.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'creators': [ + f'Tool: SCANOSS-PY: {__version__}', + f'Person: {getpass.getuser()}', + 'Organization: SCANOSS' + ], + 'comment': 'SBOM Build information - SBOM Type: Build', + } + + def _process_packages(self, raw_data: dict, spdx_document: dict): + """Process packages and add them to the SPDX document.""" + lic_refs = set() + + for purl, comp in raw_data.items(): + package_info = self._create_package_info(purl, comp, lic_refs) + spdx_document['packages'].append(package_info) + spdx_document['documentDescribes'].append(package_info['SPDXID']) + + self._process_license_refs(lic_refs, spdx_document) + + def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict: + """Create package information for SPDX document.""" + lic_text = self._process_package_licenses(comp.get('licenses', []), lic_refs) + comp_ver = comp.get('version') + purl_ver = f'{purl}@{comp_ver}' + purl_hash = hashlib.md5(purl_ver.encode('utf-8')).hexdigest() + + return { + 'name': comp.get('component'), + 'SPDXID': f'SPDXRef-{purl_hash}', + 'versionInfo': comp_ver, + 'downloadLocation': comp.get('download_url') or comp.get('url'), + 'homepage': comp.get('url', ''), + 'licenseDeclared': lic_text, + 'licenseConcluded': 'NOASSERTION', + 'filesAnalyzed': False, + 'copyrightText': 'NOASSERTION', + 'supplier': f'Organization: {comp.get("vendor", "NOASSERTION")}', + 'externalRefs': [ { - 'name': comp_name, - 'SPDXID': purl_spdx, - 'versionInfo': comp_ver, - 'downloadLocation': 'NOASSERTION', # TODO Add actual download location - 'homepage': comp.get('url', ''), - 'licenseDeclared': lic_text, - 'licenseConcluded': 'NOASSERTION', - 'filesAnalyzed': False, - 'copyrightText': 'NOASSERTION', - 'supplier': supplier, - 'externalRefs': [ - {'referenceCategory': 'PACKAGE-MANAGER', 'referenceLocator': purl_ver, 'referenceType': 'purl'} - ], + 'referenceCategory': 'PACKAGE-MANAGER', + 'referenceLocator': purl_ver, + 'referenceType': 'purl' } - ) - # End purls for loop - for lic_ref in lic_refs: # Insert all the non-SPDX license references - source = '' - match = re.search(r'^LicenseRef-(scancode-|scanoss-|)(\S+)$', lic_ref, re.IGNORECASE) - if match: - source = match.group(1).replace('-', '') # source for the custom license - name = match.group(2) # license name (without references, etc.) - else: - name = lic_ref - name = name.replace('-', ' ') - source = f' by {source}.' if source else '.' - data['hasExtractedLicensingInfos'].append( + ], + 'checksums': [ { - 'licenseId': lic_ref, - 'name': name, - 'extractedText': 'Detected license, please review component source code.', - 'comment': f'Detected license{source}', + 'algorithm': 'MD5', + 'checksumValue': comp.get('url_hash') or '0' * 32 } - ) - # End license refs for loop - file = sys.stdout + ], + } + + def _process_package_licenses(self, licenses: list, lic_refs: set) -> str: + """Process licenses and return license text.""" + if not licenses: + return 'NOASSERTION' + + lic_set = set() + for lic in licenses: + lc_id = lic.get('id') + if lc_id: + self._process_license_id(lc_id, lic_refs, lic_set) + + return self._format_license_text(lic_set) + + def _process_license_id(self, lc_id: str, lic_refs: set, lic_set: set): + """Process individual license ID.""" + spdx_id = self.get_spdx_license_id(lc_id) + if not spdx_id: + if not lc_id.startswith('LicenseRef'): + lc_id = f'LicenseRef-{lc_id}' + lic_refs.add(lc_id) + lic_set.add(spdx_id if spdx_id else lc_id) + + def _format_license_text(self, lic_set: set) -> str: + """Format the license text with proper syntax.""" + if not lic_set: + return 'NOASSERTION' + + lic_text = ' AND '.join(lic_set) + if len(lic_set) > 1: + lic_text = f'({lic_text})' + return lic_text + + def _process_license_refs(self, lic_refs: set, spdx_document: dict): + """Process and add license references to the document.""" + for lic_ref in lic_refs: + license_info = self._parse_license_ref(lic_ref) + spdx_document['hasExtractedLicensingInfos'].append(license_info) + + def _parse_license_ref(self, lic_ref: str) -> dict: + """Parse license reference and create info dictionary.""" + source, name = self._extract_license_info(lic_ref) + source_text = f' by {source}.' if source else '.' + + return { + 'licenseId': lic_ref, + 'name': name.replace('-', ' '), + 'extractedText': 'Detected license, please review component source code.', + 'comment': f'Detected license{source_text}', + } + + def _extract_license_info(self, lic_ref: str): + """Extract source and name from license reference.""" + match = re.search(r'^LicenseRef-(scancode-|scanoss-|)(\S+)$', lic_ref, re.IGNORECASE) + if match: + source = match.group(1).replace('-', '') + name = match.group(2) + else: + source = '' + name = lic_ref + return source, name + + def _write_output(self, data: dict, output_file: str = None) -> bool: + """Write the SPDX document to output.""" + try: + file = self._get_output_file(output_file) + print(json.dumps(data, indent=2), file=file) + if output_file: + file.close() + return True + except Exception as e: + self.print_stderr(f'Error writing output: {str(e)}') + return False + + def _get_output_file(self, output_file: str = None): + """Get the appropriate output file handle.""" if not output_file and self.output_file: output_file = self.output_file - if output_file: - file = open(output_file, 'w') - print(json.dumps(data, indent=2), file=file) - if output_file: - file.close() - return True + return open(output_file, 'w') if output_file else sys.stdout def produce_from_str(self, json_str: str, output_file: str = None) -> bool: """ diff --git a/tests/test_spdxlite.py b/tests/test_spdxlite.py new file mode 100644 index 00000000..762dc58a --- /dev/null +++ b/tests/test_spdxlite.py @@ -0,0 +1,70 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +import os +import tempfile +import unittest + +from scanoss.spdxlite import SpdxLite + + +class MyTestCase(unittest.TestCase): + """ + Exercise the SpdxLite class + """ + def testSpdxLite(self): + temp_dir = tempfile.gettempdir() + spdx_lite_output = os.path.join(temp_dir, "spdxlite.json") + test_data_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(test_data_dir, 'data', file_name) + spdx_lite = SpdxLite(debug = False, output_file=spdx_lite_output) + spdx_lite.produce_from_file(input_file_name) + md5_length = 32 + # Read data using absolute path + with open(spdx_lite_output, 'r') as f: + parsed_data = json.load(f) + spdx_version = parsed_data.get("spdxVersion") + spdx_id = parsed_data.get("SPDXID") + name = parsed_data.get("name") + organization = parsed_data.get("creationInfo",{}).get('creators')[2] + creation_info_comment = parsed_data.get("creationInfo", {}).get('comment') + document_describes = parsed_data.get("documentDescribes") + packages = parsed_data.get("packages") + + self.assertEqual(spdx_version, "SPDX-2.2") + self.assertEqual(spdx_id, "SPDXRef-DOCUMENT") + self.assertEqual(name, "SCANOSS-SBOM") + self.assertEqual(organization, "Organization: SCANOSS") + self.assertEqual(creation_info_comment, "SBOM Build information - SBOM Type: Build") + self.assertEqual(len(document_describes), 5) + self.assertEqual(len(packages), 5) + + for package in packages: + for checksum in package.get("checksums", []): + self.assertEqual(checksum.get("algorithm"), "MD5") #Check all algorithms be MD5 + self.assertEqual(len(checksum.get("checksumValue")), md5_length) #Check checksum length value be 32 + + + os.remove(spdx_lite_output) #Removes tmp spdxlite.json file \ No newline at end of file From 46aea97ecf5967c0f647eb2c8f9b4e95975ed050 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Wed, 26 Feb 2025 17:01:25 -0300 Subject: [PATCH 276/489] Fixes bug on provenance command * bug:SP-2169 Fixes bug on provenance command --- CHANGELOG.md | 7 +- pyproject.toml | 2 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 203 ++++++++++++++++++++++------------------ 4 files changed, 122 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53db4de0..45b7d805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.2] - 2025-02-26 +### Fixed +- Fixed provenance command + ## [1.20.1] - 2025-02-18 ### Added - Enhanced SPDX Lite report to achieve Telco compliance @@ -461,4 +465,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.19.4]: https://github.com/scanoss/scanoss.py/compare/v1.19.3...v1.19.4 [1.19.5]: https://github.com/scanoss/scanoss.py/compare/v1.19.4...v1.19.5 [1.20.0]: https://github.com/scanoss/scanoss.py/compare/v1.19.5...v1.20.0 -[1.20.1]: https://github.com/scanoss/scanoss.py/compare/v1.20.0...v1.20.1 \ No newline at end of file +[1.20.1]: https://github.com/scanoss/scanoss.py/compare/v1.20.0...v1.20.1 +[1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8cde7b9a..ef531e04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ select = ["E", "F", "I", "PL"] line-length = 120 # Assume Python 3.7+ target-version = "py37" -exclude = ["tests/*", "test_*.py"] +exclude = ["tests/*", "test_*.py", "src/scanoss/cli.py"] [tool.ruff.format] quote-style = "single" diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 733f4109..615df418 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.1' +__version__ = '1.20.2' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index aa4e0ba3..eaad1df7 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -45,6 +45,11 @@ from .threadeddependencies import SCOPE from .utils.file import validate_json_file +DEFAULT_POST_SIZE = 32 +DEFAULT_TIMEOUT = 180 +MIN_TIMEOUT_VALUE = 5 +DEFAULT_RETRY = 5 +PYTHON3_OR_LATER = 3 def print_stderr(*args, **kwargs): """ @@ -53,7 +58,7 @@ def print_stderr(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def setup_args() -> None: +def setup_args() -> None: # noqa: PLR0915 """ Setup all the command line arguments for processing """ @@ -114,24 +119,26 @@ def setup_args() -> None: '--post-size', '-P', type=int, - default=32, + default=DEFAULT_POST_SIZE, help='Number of kilobytes to limit the post to while scanning (optional - default 32)', ) p_scan.add_argument( '--timeout', '-M', type=int, - default=180, + default=DEFAULT_TIMEOUT, help='Timeout (in seconds) for API communication (optional - default 180)', ) p_scan.add_argument( - '--retry', '-R', type=int, default=5, help='Retry limit for API communication (optional - default 5)' + '--retry', '-R', type=int, default=DEFAULT_RETRY, + help='Retry limit for API communication (optional - default 5)' ) p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') p_scan.add_argument( - '--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' + '--sc-command', type=str, + help='Scancode command and path if required (optional - default scancode).' ) p_scan.add_argument( '--sc-timeout', @@ -290,7 +297,7 @@ def setup_args() -> None: description=f'Show Provenance findings: {__version__}', help='Retrieve provenance for the given components', ) - c_provenance.set_defaults(func=c_provenance) + c_provenance.set_defaults(func=comp_provenance) # Component Sub-command: component search c_search = comp_sub.add_parser( @@ -321,7 +328,7 @@ def setup_args() -> None: c_versions.set_defaults(func=comp_versions) # Common purl Component sub-command options - for p in [c_crypto, c_vulns, c_semgrep]: + for p in [c_crypto, c_vulns, c_semgrep, c_provenance]: p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') # Common Component sub-command options @@ -524,7 +531,7 @@ def setup_args() -> None: p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.') # Global Scan/GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: p.add_argument( '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' ) @@ -550,7 +557,7 @@ def setup_args() -> None: ) # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: p.add_argument( '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' ) @@ -579,6 +586,7 @@ def setup_args() -> None: p_results, p_undeclared, p_copyleft, + c_provenance ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') @@ -587,21 +595,14 @@ def setup_args() -> None: args = parser.parse_args() if args.version: ver(parser, args) - exit(0) + sys.sys.exit(0) if not args.subparser: parser.print_help() # No sub command subcommand, print general help - exit(1) + sys.exit(1) elif ( - args.subparser == 'utils' - or args.subparser == 'ut' - or args.subparser == 'component' - or args.subparser == 'comp' - or args.subparser == 'inspect' - or args.subparser == 'insp' - or args.subparser == 'ins' - ) and not args.subparsercmd: + args.subparser in {'utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins'} and not args.subparsercmd): parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed - exit(1) + sys.exit(1) args.func(parser, args) # Execute the function associated with the sub-command @@ -634,7 +635,7 @@ def file_count(parser, args): if not args.scan_dir: print_stderr('Please specify a folder') parser.parse_args([args.subparser, '-h']) - exit(1) + sys.exit(1) scan_output: str = None if args.output: scan_output = args.output @@ -649,12 +650,12 @@ def file_count(parser, args): ) if not os.path.exists(args.scan_dir): print_stderr(f'Error: Folder specified does not exist: {args.scan_dir}.') - exit(1) + sys.exit(1) if os.path.isdir(args.scan_dir): counter.count_files(args.scan_dir) else: print_stderr(f'Error: Path specified is not a folder: {args.scan_dir}.') - exit(1) + sys.exit(1) def wfp(parser, args): @@ -670,7 +671,7 @@ def wfp(parser, args): if not args.scan_dir and not args.stdin: print_stderr('Please specify a file/folder or STDIN (--stdin)') parser.parse_args([args.subparser, '-h']) - exit(1) + sys.exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: print_stderr('Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') scan_output: str = None @@ -686,7 +687,7 @@ def wfp(parser, args): scan_settings.load_json_file(args.settings) except ScanossSettingsError as e: print_stderr(f'Error: {e}') - exit(1) + sys.exit(1) scan_options = 0 if args.skip_snippets else ScanType.SCAN_SNIPPETS.value # Skip snippet generation or not scanner = Scanner( @@ -713,17 +714,17 @@ def wfp(parser, args): elif args.scan_dir: if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') - exit(1) + sys.exit(1) if os.path.isdir(args.scan_dir): scanner.wfp_folder(args.scan_dir, scan_output) elif os.path.isfile(args.scan_dir): scanner.wfp_file(args.scan_dir, scan_output) else: print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') - exit(1) + sys.exit(1) else: print_stderr('No action found to process') - exit(1) + sys.exit(1) def get_scan_options(args): @@ -754,11 +755,11 @@ def get_scan_options(args): print_stderr('Scan Dependencies') if scan_options <= 0: print_stderr(f'Error: No valid scan options configured: {scan_options}') - exit(1) + sys.exit(1) return scan_options -def scan(parser, args): +def scan(parser, args): # noqa: PLR0912, PLR0915 """ Run the "scan" sub-command Parameters @@ -773,17 +774,17 @@ def scan(parser, args): 'Please specify a file/folder, files (--files), fingerprint (--wfp), dependency (--dep), or STDIN (--stdin)' ) parser.parse_args([args.subparser, '-h']) - exit(1) + sys.exit(1) if args.pac and args.proxy: print_stderr('Please specify one of --proxy or --pac, not both') parser.parse_args([args.subparser, '-h']) - exit(1) + sys.exit(1) if args.identify and args.settings: print_stderr('ERROR: Cannot specify both --identify and --settings options.') - exit(1) + sys.exit(1) if args.settings and args.skip_settings_file: print_stderr('ERROR: Cannot specify both --settings and --skip-file-settings options.') - exit(1) + sys.exit(1) # Figure out which settings (if any) to load before processing scan_settings = None if not args.skip_settings_file: @@ -803,15 +804,15 @@ def scan(parser, args): ) except ScanossSettingsError as e: print_stderr(f'Error: {e}') - exit(1) + sys.exit(1) if args.dep: if not os.path.exists(args.dep) or not os.path.isfile(args.dep): print_stderr(f'Specified --dep file does not exist or is not a file: {args.dep}') - exit(1) + sys.exit(1) result = validate_json_file(args.dep) if not result.is_valid: print_stderr(f'Error: Dependency file is not valid: {result.error}') - exit(1) + sys.exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: print_stderr('Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') @@ -832,11 +833,11 @@ def scan(parser, args): print_stderr('Scanning all hidden files/folders...') if args.skip_snippets: print_stderr('Skipping snippets...') - if args.post_size != 32: + if args.post_size != DEFAULT_POST_SIZE: print_stderr(f'Changing scanning POST size to: {args.post_size}k...') - if args.timeout != 180: + if args.timeout != DEFAULT_TIMEOUT: print_stderr(f'Changing scanning POST timeout to: {args.timeout}...') - if args.retry != 5: + if args.retry != DEFAULT_RETRY: print_stderr(f'Changing scanning POST retry to: {args.retry}...') if args.obfuscate: print_stderr('Obfuscating file fingerprints...') @@ -853,7 +854,7 @@ def scan(parser, args): if flags: print_stderr(f'Using flags {flags}...') elif not args.quiet: - if args.timeout < 5: + if args.timeout < MIN_TIMEOUT_VALUE: print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') if args.retry < 0: print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.') @@ -863,7 +864,7 @@ def scan(parser, args): args.no_wfp_output = True if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') - exit(1) + sys.exit(1) pac_file = get_pac_file(args.pac) scan_options = get_scan_options(args) # Figure out what scanning options we have @@ -906,22 +907,22 @@ def scan(parser, args): if args.wfp: if not scanner.is_file_or_snippet_scan(): print_stderr(f'Error: Cannot specify WFP scanning if file/snippet options are disabled ({scan_options})') - exit(1) + sys.exit(1) if scanner.is_dependency_scan() and not args.dep: print_stderr('Error: Cannot specify WFP & Dependency scanning without a dependency file (--dep)') - exit(1) + sys.exit(1) scanner.scan_wfp_with_options(args.wfp, args.dep) elif args.stdin: contents = sys.stdin.buffer.read() if not scanner.scan_contents(args.stdin, contents): - exit(1) + sys.exit(1) elif args.files: if not scanner.scan_files_with_options(args.files, args.dep, scanner.winnowing.file_map): - exit(1) + sys.exit(1) elif args.scan_dir: if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') - exit(1) + sys.exit(1) if os.path.isdir(args.scan_dir): if not scanner.scan_folder_with_options( args.scan_dir, @@ -931,7 +932,7 @@ def scan(parser, args): args.dep_scope_inc, args.dep_scope_exc, ): - exit(1) + sys.exit(1) elif os.path.isfile(args.scan_dir): if not scanner.scan_file_with_options( args.scan_dir, @@ -941,23 +942,24 @@ def scan(parser, args): args.dep_scope_inc, args.dep_scope_exc, ): - exit(1) + sys.exit(1) else: print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') - exit(1) + sys.exit(1) elif args.dep: if not args.dependencies_only: print_stderr( - 'Error: No file or folder specified to scan. Please add --dependencies-only to decorate dependency file only.' + 'Error: No file or folder specified to scan.' + ' Please add --dependencies-only to decorate dependency file only.' ) - exit(1) + sys.exit(1) if not scanner.scan_folder_with_options( '.', args.dep, scanner.winnowing.file_map, args.dep_scope, args.dep_scope_inc, args.dep_scope_exc ): - exit(1) + sys.exit(1) else: print_stderr('No action found to process') - exit(1) + sys.exit(1) def dependency(parser, args): @@ -973,10 +975,10 @@ def dependency(parser, args): if not args.scan_dir: print_stderr('Please specify a file/folder') parser.parse_args([args.subparser, '-h']) - exit(1) + sys.exit(1) if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') - exit(1) + sys.exit(1) scan_output: str = None if args.output: scan_output = args.output @@ -986,7 +988,7 @@ def dependency(parser, args): debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout ) if not sc_deps.get_dependencies(what_to_scan=args.scan_dir, result_output=scan_output): - exit(1) + sys.exit(1) def convert(parser, args): @@ -1002,7 +1004,7 @@ def convert(parser, args): if not args.input: print_stderr('Please specify an input file to convert') parser.parse_args([args.subparser, '-h']) - exit(1) + sys.exit(1) success = False if args.format == 'cyclonedx': if not args.quiet: @@ -1022,7 +1024,7 @@ def convert(parser, args): else: print_stderr(f'ERROR: Unknown output format (--format): {args.format}') if not success: - exit(1) + sys.exit(1) def inspect_copyleft(parser, args): @@ -1038,7 +1040,7 @@ def inspect_copyleft(parser, args): if args.input is None: print_stderr('Please specify an input file to inspect') parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) + sys.exit(1) output: str = None if args.output: output = args.output @@ -1062,7 +1064,7 @@ def inspect_copyleft(parser, args): explicit=args.explicit, ) status, _ = i_copyleft.run() - sys.exit(status) + sys.sys.exit(status) def inspect_undeclared(parser, args): @@ -1078,7 +1080,7 @@ def inspect_undeclared(parser, args): if args.input is None: print_stderr('Please specify an input file to inspect') parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) + sys.exit(1) output: str = None if args.output: output = args.output @@ -1099,7 +1101,7 @@ def inspect_undeclared(parser, args): sbom_format=args.sbom_format, ) status, _ = i_undeclared.run() - sys.exit(status) + sys.sys.exit(status) def utils_certloc(*_): @@ -1112,7 +1114,7 @@ def utils_certloc(*_): print(f'CA Cert File: {certifi.where()}') -def utils_cert_download(_, args): +def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912 """ Run the "utils cert-download" sub-command :param _: ignore/unused @@ -1139,13 +1141,13 @@ def utils_cert_download(_, args): certs = conn.get_peer_cert_chain() for index, cert in enumerate(certs): cert_components = dict(cert.get_subject().get_components()) - if sys.version_info[0] >= 3: + if sys.version_info[0] >= PYTHON3_OR_LATER: cn = cert_components.get(b'CN') else: cn = cert_components.get('CN') if not args.quiet: print_stderr(f'Certificate {index} - CN: {cn}') - if sys.version_info[0] >= 3: + if sys.version_info[0] >= PYTHON3_OR_LATER: print( (crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file ) # Print the downloaded PEM certificate @@ -1155,7 +1157,7 @@ def utils_cert_download(_, args): print_stderr(f'ERROR: Exception ({e.__class__.__name__}) Downloading certificate from {hostname}:{port} - {e}.') if args.debug: traceback.print_exc() - exit(1) + sys.exit(1) else: if args.output: if args.debug: @@ -1173,11 +1175,11 @@ def utils_pac_proxy(_, args): if not args.pac: print_stderr('Error: No pac file option specified.') - exit(1) + sys.exit(1) pac_file = get_pac_file(args.pac) if pac_file is None: print_stderr(f'No proxy configuration for: {args.pac}') - exit(1) + sys.exit(1) resolver = ProxyResolver(pac_file) proxies = resolver.get_proxy_for_requests(args.url) print(f'Proxies: {proxies}\n') @@ -1197,14 +1199,14 @@ def get_pac_file(pac: str): pac_local = pac.strip('file://') if not os.path.exists(pac_local): print_stderr(f'Error: PAC file does not exist: {pac_local}.') - exit(1) + sys.exit(1) with open(pac_local) as pf: pac_file = pypac.get_pac(js=pf.read()) elif pac.startswith('http'): pac_file = pypac.get_pac(url=pac) else: print_stderr(f'Error: Unknown PAC file option: {pac}. Should be one of "auto", "file://", "https://"') - exit(1) + sys.exit(1) return pac_file @@ -1221,10 +1223,10 @@ def comp_crypto(parser, args): if (not args.purl and not args.input) or (args.purl and args.input): print_stderr('Please specify an input file or purl to decorate (--purl or --input)') parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) + sys.exit(1) if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') - exit(1) + sys.exit(1) pac_file = get_pac_file(args.pac) comps = Components( debug=args.debug, @@ -1239,7 +1241,7 @@ def comp_crypto(parser, args): timeout=args.timeout, ) if not comps.get_crypto_details(args.input, args.purl, args.output): - exit(1) + sys.exit(1) def comp_vulns(parser, args): @@ -1255,10 +1257,10 @@ def comp_vulns(parser, args): if (not args.purl and not args.input) or (args.purl and args.input): print_stderr('Please specify an input file or purl to decorate (--purl or --input)') parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) + sys.exit(1) if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') - exit(1) + sys.exit(1) pac_file = get_pac_file(args.pac) comps = Components( debug=args.debug, @@ -1273,7 +1275,7 @@ def comp_vulns(parser, args): timeout=args.timeout, ) if not comps.get_vulnerabilities(args.input, args.purl, args.output): - exit(1) + sys.exit(1) def comp_semgrep(parser, args): @@ -1289,10 +1291,10 @@ def comp_semgrep(parser, args): if (not args.purl and not args.input) or (args.purl and args.input): print_stderr('Please specify an input file or purl to decorate (--purl or --input)') parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) + sys.exit(1) if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') - exit(1) + sys.exit(1) pac_file = get_pac_file(args.pac) comps = Components( debug=args.debug, @@ -1307,7 +1309,7 @@ def comp_semgrep(parser, args): timeout=args.timeout, ) if not comps.get_semgrep_details(args.input, args.purl, args.output): - exit(1) + sys.exit(1) def comp_search(parser, args): @@ -1325,11 +1327,11 @@ def comp_search(parser, args): ): print_stderr('Please specify an input file or search terms (--input or --search, or --vendor or --comp.)') parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) + sys.exit(1) if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') - exit(1) + sys.exit(1) pac_file = get_pac_file(args.pac) comps = Components( debug=args.debug, @@ -1353,7 +1355,7 @@ def comp_search(parser, args): limit=args.limit, offset=args.offset, ): - exit(1) + sys.exit(1) def comp_versions(parser, args): @@ -1369,11 +1371,11 @@ def comp_versions(parser, args): if (not args.input and not args.purl) or (args.input and args.purl): print_stderr('Please specify an input file or search terms (--input or --purl.)') parser.parse_args([args.subparser, args.subparsercmd, '-h']) - exit(1) + sys.exit(1) if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') - exit(1) + sys.exit(1) pac_file = get_pac_file(args.pac) comps = Components( debug=args.debug, @@ -1388,8 +1390,31 @@ def comp_versions(parser, args): timeout=args.timeout, ) if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit): - exit(1) + sys.exit(1) +def comp_provenance(parser, args): + """ + Run the "component semgrep" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + sys.exit(1) + pac_file = get_pac_file(args.pac) + comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, + ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, + timeout=args.timeout) + if not comps.get_provenance_details(args.input, args.purl, args.output): + sys.exit(1) def results(parser, args): """ @@ -1404,13 +1429,13 @@ def results(parser, args): if not args.filepath: print_stderr('ERROR: Please specify a file containing the results') parser.parse_args([args.subparser, '-h']) - exit(1) + sys.exit(1) file_path = Path(args.filepath).resolve() if not file_path.is_file(): print_stderr(f'The specified file {args.filepath} does not exist') - exit(1) + sys.exit(1) results = Results( debug=args.debug, @@ -1426,7 +1451,7 @@ def results(parser, args): if args.has_pending: results.get_pending_identifications().present() if results.has_results(): - exit(1) + sys.exit(1) else: results.apply_filters().present() From 23096cb36e3e412e686c611a1b366879b83e0f32 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 3 Mar 2025 13:05:51 +0100 Subject: [PATCH 277/489] fix sys.call typo --- src/scanoss/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index eaad1df7..611172e5 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -595,7 +595,7 @@ def setup_args() -> None: # noqa: PLR0915 args = parser.parse_args() if args.version: ver(parser, args) - sys.sys.exit(0) + sys.exit(0) if not args.subparser: parser.print_help() # No sub command subcommand, print general help sys.exit(1) From 471a4037520b765f8c1c6a9b0c735aea9dd72248 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 3 Mar 2025 13:06:42 +0100 Subject: [PATCH 278/489] update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b7d805..04675f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.3] - 2025-03-03 +### Fixed +- Fixed cli.py typo + ## [1.20.2] - 2025-02-26 ### Fixed - Fixed provenance command @@ -466,4 +470,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.19.5]: https://github.com/scanoss/scanoss.py/compare/v1.19.4...v1.19.5 [1.20.0]: https://github.com/scanoss/scanoss.py/compare/v1.19.5...v1.20.0 [1.20.1]: https://github.com/scanoss/scanoss.py/compare/v1.20.0...v1.20.1 -[1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 \ No newline at end of file +[1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 +[1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 \ No newline at end of file From e6d141af5419842fea6655dff1de294a5ddf13d6 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 3 Mar 2025 13:06:59 +0100 Subject: [PATCH 279/489] update version --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 615df418..9d88bfbe 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.2' +__version__ = '1.20.3' From a3f4b7e4eb13a4c4cbff26e099ed9bfc4bd9b9d3 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 3 Mar 2025 13:19:16 +0100 Subject: [PATCH 280/489] fix: two more typos --- src/scanoss/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 611172e5..54a6db0a 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1064,7 +1064,7 @@ def inspect_copyleft(parser, args): explicit=args.explicit, ) status, _ = i_copyleft.run() - sys.sys.exit(status) + sys.exit(status) def inspect_undeclared(parser, args): @@ -1101,7 +1101,7 @@ def inspect_undeclared(parser, args): sbom_format=args.sbom_format, ) status, _ = i_undeclared.run() - sys.sys.exit(status) + sys.exit(status) def utils_certloc(*_): From e4c648e1aabffe4991c2d7e0bdb82409fce15a16 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 5 Mar 2025 10:02:45 +0100 Subject: [PATCH 281/489] chore: update dockerfile to use python 3.10-slim --- CHANGELOG.md | 7 ++++++- Dockerfile | 2 +- src/scanoss/__init__.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04675f99..878cadcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.4] - 2025-03-05 +### Modified +- Updated Dockerfile to use Python 3.10-slim + ## [1.20.3] - 2025-03-03 ### Fixed - Fixed cli.py typo @@ -471,4 +475,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.0]: https://github.com/scanoss/scanoss.py/compare/v1.19.5...v1.20.0 [1.20.1]: https://github.com/scanoss/scanoss.py/compare/v1.20.0...v1.20.1 [1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 -[1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 \ No newline at end of file +[1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 +[1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1ba624dd..fc066e50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM python:3.10-slim-buster AS base +FROM --platform=$BUILDPLATFORM python:3.10-slim AS base LABEL maintainer="SCANOSS " LABEL org.opencontainers.image.source=https://github.com/scanoss/scanoss.py diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 9d88bfbe..3340d95f 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.3' +__version__ = '1.20.4' From aa134317463abfd39799485b41727483760ac9ab Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Fri, 14 Mar 2025 08:45:21 -0300 Subject: [PATCH 282/489] Bug/jortiz/spdxlite output (#107) * bug:SP-2204 Adds condition to skip scancode licenses * chore:SP-2241 Improve documentation on spdxlite.py file * chore: Updates CHANGELOG file * chore: Upgrade version to v1.20.5 * chore:SP-2243 Uses packageurl-python library for reference locator field --------- Co-authored-by: Jeronimo Ortiz <166400360+ortizjeronimo@users.noreply.github.com> --- CHANGELOG.md | 9 +- src/scanoss/__init__.py | 2 +- src/scanoss/spdxlite.py | 311 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 290 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 878cadcb..fd7f9a3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.5] - 2025-03-13 +### Added +- Improved documentation on spdxlite.py file +### Modified +- Added validation for license source on SPDXLite output format + ## [1.20.4] - 2025-03-05 ### Modified - Updated Dockerfile to use Python 3.10-slim @@ -476,4 +482,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.1]: https://github.com/scanoss/scanoss.py/compare/v1.20.0...v1.20.1 [1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 [1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 -[1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 \ No newline at end of file +[1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 +[1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 3340d95f..00f19ec1 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.4' +__version__ = '1.20.5' diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index b24b178d..fe864eba 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -31,6 +31,7 @@ import sys import importlib_resources +from packageurl import PackageURL from . import __version__ @@ -78,15 +79,31 @@ def parse(self, data: json): return self._process_files(data) def _process_files(self, data: json) -> dict: - """Process each file in the data and build summary.""" + """ + Process raw results and build a component summary. + + Args: + data: JSON data containing raw results + + Returns: + dict: The built summary dictionary + """ summary = {} for file_path in data: file_details = data.get(file_path) - self._process_file_entries(file_path, file_details, summary) + # summary is passed by reference and modified inside the function + self._process_entries(file_path, file_details, summary) return summary - def _process_file_entries(self, file_path: str, file_details: list, summary: dict): - """Process entries for a single file.""" + def _process_entries(self, file_path: str, file_details: list, summary: dict): + """ + Process entries for a single file. + + Args: + file_path: Path to the file being processed + file_details: Results of the file + summary: Reference to summary dictionary that will be modified in place + """ for entry in file_details: id_details = entry.get('id') if not id_details or id_details == 'none': @@ -95,10 +112,17 @@ def _process_file_entries(self, file_path: str, file_details: list, summary: dic if id_details == 'dependency': self._process_dependency_entry(file_path, entry, summary) else: - self._process_normal_entry(file_path, entry, summary) + self._process_file_entry(file_path, entry, summary) def _process_dependency_entry(self, file_path: str, entry: dict, summary: dict): - """Process a dependency type entry.""" + """ + Process a dependency type entry. + + Args: + file_path: Path to the file being processed + entry: The dependency entry to process + summary: Reference to summary dictionary that will be modified in place + """ dependencies = entry.get('dependencies') if not dependencies: self.print_stderr(f'Warning: No Dependencies found for {file_path}') @@ -108,11 +132,18 @@ def _process_dependency_entry(self, file_path: str, entry: dict, summary: dict): purl = dep.get('purl') if not self._is_valid_purl(file_path, dep, purl, summary): continue - + # Modifying the summary dictionary directly as it's passed by reference summary[purl] = self._create_dependency_summary(dep) - def _process_normal_entry(self, file_path: str, entry: dict, summary: dict): - """Process a normal file type entry.""" + def _process_file_entry(self, file_path: str, entry: dict, summary: dict): + """ + Process file entry. + + Args: + file_path: Path to the file being processed + entry: Process file match entry + summary: Reference to summary dictionary that will be modified in place + """ purls = entry.get('purl') if not purls: self.print_stderr(f'Purl block missing for {file_path}') @@ -122,10 +153,21 @@ def _process_normal_entry(self, file_path: str, entry: dict, summary: dict): if not self._is_valid_purl(file_path, entry, purl, summary): return - summary[purl] = self._create_normal_summary(entry) + summary[purl] = self._create_file_summary(entry) def _is_valid_purl(self, file_path: str, entry: dict, purl: str, summary: dict) -> bool: - """Check if PURL is valid and not already processed.""" + """ + Check if purl is valid and not already processed. + + Args: + file_path: Path to the file being processed + entry: The entry containing the PURL + purl: The PURL to validate + summary: Reference to summary dictionary to check for existing entries + + Returns: + bool: True if purl is valid and not already processed + """ if not purl: self.print_stderr(f'Warning: No PURL found for {file_path}: {entry}') return False @@ -137,15 +179,37 @@ def _is_valid_purl(self, file_path: str, entry: dict, purl: str, summary: dict) return True def _create_dependency_summary(self, dep: dict) -> dict: - """Create summary for dependency entry.""" + """ + Create summary for dependency entry. + + This method extracts relevant fields from a dependency entry and creates a + standardized summary dictionary. It handles fields like component, version, + and URL, with special processing for licenses. + + Args: + dep (dict): The dependency entry containing component information + + Returns: + dict: A new summary dictionary containing the extracted and processed fields + """ summary = {} for field in ['component', 'version', 'url']: summary[field] = dep.get(field, '') summary['licenses'] = self._process_licenses(dep.get('licenses')) return summary - def _create_normal_summary(self, entry: dict) -> dict: - """Create summary for normal file entry.""" + def _create_file_summary(self, entry: dict) -> dict: + """ + Create summary for file entry. + + This method extracts set of fields from file entry and creates a standardized summary dictionary. + + Args: + entry (dict): The file entry containing the metadata to summarize + + Returns: + dict: A new summary dictionary containing all extracted and processed fields + """ summary = {} fields = ['id', 'vendor', 'component', 'version', 'latest', 'url', 'url_hash', 'download_url'] @@ -155,7 +219,22 @@ def _create_normal_summary(self, entry: dict) -> dict: return summary def _process_licenses(self, licenses: list) -> list: - """Process license information and remove duplicates.""" + """ + Process license information and remove duplicates. + + This method filters license information to include only licenses from trusted sources + ('component_declared' or 'license_file') and removes any duplicate license names. + The result is a simplified list of license dictionaries containing only the 'id' field. + + Args: + licenses (list): A list of license dictionaries, each containing at least 'name' + and 'source' fields. Can be None or empty. + + Returns: + list: A filtered and deduplicated list of license dictionaries, where each + dictionary contains only an 'id' field matching the original license name. + Returns an empty list if input is None or empty. + """ if not licenses: return [] @@ -164,6 +243,9 @@ def _process_licenses(self, licenses: list) -> list: for license_info in licenses: name = license_info.get('name') + source = license_info.get('source') + if source not in ("component_declared", "license_file", "file_header"): + continue if name and name not in seen_names: processed_licenses.append({'id': name}) seen_names.add(name) @@ -205,7 +287,30 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: return self._write_output(spdx_document, output_file) def _create_base_document(self, raw_data: dict) -> dict: - """Create the base SPDX document structure.""" + """ + Create the base SPDX document structure. + + This method initializes a new SPDX document with standard fields required by + the SPDX 2.2 specification. It generates a unique document namespace using + a hash of the raw data and current timestamp. + + Args: + raw_data (dict): The raw component data used to create a unique identifier + for the document namespace + + Returns: + dict: A dictionary containing the base SPDX document structure with the + following fields: + - spdxVersion: The SPDX specification version + - dataLicense: The license for the SPDX document itself + - SPDXID: The document's unique identifier + - name: The name of the SBOM + - creationInfo: Information about when and how the document was created + - documentNamespace: A unique URI for this document + - documentDescribes: List of packages described (initially empty) + - hasExtractedLicensingInfos: List of licenses (initially empty) + - packages: List of package information (initially empty) + """ now = datetime.datetime.utcnow() md5hex = hashlib.md5(f'{raw_data}-{now}'.encode('utf-8')).hexdigest() @@ -222,7 +327,23 @@ def _create_base_document(self, raw_data: dict) -> dict: } def _create_creation_info(self, timestamp: datetime.datetime) -> dict: - """Create the creation info section.""" + """ + Create the creation info section of an SPDX document. + + This method generates the creation information required by the SPDX specification, + including timestamps, creator information, and document type. + + Args: + timestamp (datetime.datetime): The UTC timestamp representing when the + document was created + + Returns: + dict: A dictionary containing creation information with the following fields: + - created: ISO 8601 formatted timestamp + - creators: List of entities involved in creating the document + (tool, person, and organization) + - comment: Additional information about the SBOM type + """ return { 'created': timestamp.strftime('%Y-%m-%dT%H:%M:%SZ'), 'creators': [ @@ -234,7 +355,25 @@ def _create_creation_info(self, timestamp: datetime.datetime) -> dict: } def _process_packages(self, raw_data: dict, spdx_document: dict): - """Process packages and add them to the SPDX document.""" + """ + Process packages and add them to the SPDX document. + + This method iterates through the raw component data, creates package information + for each component, and adds them to the SPDX document. It also collects + license references to be processed separately. + + Args: + raw_data (dict): Dictionary of package data indexed by PURL + (Package URL identifiers) + spdx_document (dict): Reference to the SPDX document being built, + which will be modified in place + + Note: + This method modifies the spdx_document dictionary in place by: + 1. Adding package information to the 'packages' list + 2. Adding package SPDXIDs to the 'documentDescribes' list + 3. Indirectly populating 'hasExtractedLicensingInfos' via _process_license_refs() + """ lic_refs = set() for purl, comp in raw_data.items(): @@ -245,7 +384,36 @@ def _process_packages(self, raw_data: dict, spdx_document: dict): self._process_license_refs(lic_refs, spdx_document) def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict: - """Create package information for SPDX document.""" + """ + Create package information for SPDX document. + + This method generates a complete package information entry following the SPDX + specification format. It creates a unique identifier for the package based on + its PURL and version, processes license information, and formats all required + fields for the SPDX document. + + Args: + purl (str): Package URL identifier for the component + comp (dict): Component information dictionary containing metadata like + component name, version, URLs, and license information + lic_refs (set): Reference to a set that will be populated with license + references found in this package. This set is modified in place. + + Returns: + dict: A dictionary containing all required SPDX package fields including: + - name: Component name + - SPDXID: Unique identifier for this package within the document + - versionInfo: Component version + - downloadLocation: URL where the package can be downloaded + - homepage: Component homepage URL + - licenseDeclared: Formatted license expression + - licenseConcluded: NOASSERTION as automated conclusion isn't possible + - filesAnalyzed: False as files are not individually analyzed + - copyrightText: NOASSERTION as copyright text isn't available + - supplier: Organization name from vendor information + - externalRefs: Package URL reference for package manager integration + - checksums: MD5 hash of the package if available + """ lic_text = self._process_package_licenses(comp.get('licenses', []), lic_refs) comp_ver = comp.get('version') purl_ver = f'{purl}@{comp_ver}' @@ -265,7 +433,7 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict: 'externalRefs': [ { 'referenceCategory': 'PACKAGE-MANAGER', - 'referenceLocator': purl_ver, + 'referenceLocator': PackageURL.from_string(purl_ver).to_string(), 'referenceType': 'purl' } ], @@ -278,20 +446,47 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict: } def _process_package_licenses(self, licenses: list, lic_refs: set) -> str: - """Process licenses and return license text.""" + """ + Process licenses and return license text formatted for SPDX. + + This method processes a list of license objects, extracts valid license IDs, + converts them to SPDX format, and combines them into a properly formatted + license expression. + + Args: + licenses (list): List of license dictionaries, each containing at least + an 'id' field + lic_refs (set): Reference to a set that will collect license references. + This set is modified in place. + + Returns: + str: A formatted license expression string following SPDX syntax. + Returns 'NOASSERTION' if no valid licenses are found. + """ if not licenses: return 'NOASSERTION' lic_set = set() for lic in licenses: lc_id = lic.get('id') - if lc_id: - self._process_license_id(lc_id, lic_refs, lic_set) + self._process_license_id(lc_id, lic_refs, lic_set) return self._format_license_text(lic_set) def _process_license_id(self, lc_id: str, lic_refs: set, lic_set: set): - """Process individual license ID.""" + """ + Process individual license ID and add to appropriate sets. + + This method attempts to convert a license ID to its SPDX equivalent. + If not found in the SPDX license list, it's formatted as a LicenseRef + and added to the license references set. + + Args: + lc_id (str): The license ID to process + lic_refs (set): Reference to a set that collects license references + for later processing. Modified in place. + lic_set (set): Reference to a set collecting all license IDs for + """ spdx_id = self.get_spdx_license_id(lc_id) if not spdx_id: if not lc_id.startswith('LicenseRef'): @@ -300,7 +495,20 @@ def _process_license_id(self, lc_id: str, lic_refs: set, lic_set: set): lic_set.add(spdx_id if spdx_id else lc_id) def _format_license_text(self, lic_set: set) -> str: - """Format the license text with proper syntax.""" + """ + Format the license text with proper SPDX syntax. + + This method combines multiple license IDs with the 'AND' operator + according to SPDX specification rules. If multiple licenses are present, + the expression is enclosed in parentheses. + + Args: + lic_set (set): Set of license IDs to format + + Returns: + str: A properly formatted SPDX license expression. + Returns 'NOASSERTION' if the set is empty. + """ if not lic_set: return 'NOASSERTION' @@ -310,13 +518,44 @@ def _format_license_text(self, lic_set: set) -> str: return lic_text def _process_license_refs(self, lic_refs: set, spdx_document: dict): - """Process and add license references to the document.""" + """ + Process and add license references to the SPDX document. + + This method processes each license reference in the provided set + and adds corresponding license information to the SPDX document's + extracted licensing information section. + + Args: + lic_refs (set): Set of license references to process + spdx_document (dict): Reference to the SPDX document being built, + which will be modified in place + + Note: + This method modifies the spdx_document dictionary in place by adding + entries to the 'hasExtractedLicensingInfos' list. + """ for lic_ref in lic_refs: license_info = self._parse_license_ref(lic_ref) spdx_document['hasExtractedLicensingInfos'].append(license_info) def _parse_license_ref(self, lic_ref: str) -> dict: - """Parse license reference and create info dictionary.""" + """ + Parse license reference and create info dictionary for SPDX document. + + This method extracts information from a license reference identifier + and formats it into the structure required by the SPDX specification + for extracted licensing information. + + Args: + lic_ref (str): License reference identifier to parse + + Returns: + dict: Dictionary containing required SPDX fields for extracted license info: + - licenseId: The unique identifier for this license + - name: A readable name for the license + - extractedText: A placeholder for the actual license text + - comment: Information about how the license was detected + """ source, name = self._extract_license_info(lic_ref) source_text = f' by {source}.' if source else '.' @@ -328,7 +567,21 @@ def _parse_license_ref(self, lic_ref: str) -> dict: } def _extract_license_info(self, lic_ref: str): - """Extract source and name from license reference.""" + """ + Extract source and name from license reference. + + This method parses a license reference string to extract the source + (e.g., scancode, scanoss) and the actual license name using regular + expressions. + + Args: + lic_ref (str): License reference identifier to parse + + Returns: + tuple: A tuple containing (source, name) where: + - source (str): The tool or system that identified the license + - name (str): The actual license name + """ match = re.search(r'^LicenseRef-(scancode-|scanoss-|)(\S+)$', lic_ref, re.IGNORECASE) if match: source = match.group(1).replace('-', '') @@ -416,8 +669,6 @@ def load_license_data_file(self, filename: str, lic_field: str = 'licenseId') -> self._spdx_licenses[lic_id_short] = lic_id if lic_name: self._spdx_lic_names[lic_name] = lic_id - # self.print_stderr(f'Licenses: {self._spdx_licenses}') - # self.print_stderr(f'Lookup: {self._spdx_lic_lookup}') return True def get_spdx_license_id(self, lic_name: str) -> str: From 1eef9651c284c3852f8d22df745aca7a8fd7615f Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 14 Mar 2025 14:59:45 +0100 Subject: [PATCH 283/489] fix: SP-2195 chunk dependency request per file + handle multiple concurrent requests --- src/scanoss/scanossgrpc.py | 144 +++++++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 53 deletions(-) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 0e21aeaa..a0a14abd 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -22,50 +22,58 @@ THE SOFTWARE. """ +import concurrent.futures +import json import os import uuid +from urllib.parse import urlparse import grpc -import json - from google.protobuf.json_format import MessageToDict, ParseDict from pypac.parser import PACFile from pypac.resolver import ProxyResolver -from urllib.parse import urlparse -from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub -from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub -from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub -from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub -from .api.provenance.v2.scanoss_provenance_pb2_grpc import ProvenanceStub -from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub -from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse -from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest, DependencyResponse -from .api.common.v2.scanoss_common_pb2 import EchoRequest, EchoResponse, StatusResponse, StatusCode, PurlRequest -from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse -from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse +from . import __version__ +from .api.common.v2.scanoss_common_pb2 import ( + EchoRequest, + EchoResponse, + PurlRequest, + StatusCode, + StatusResponse, +) from .api.components.v2.scanoss_components_pb2 import ( CompSearchRequest, CompSearchResponse, CompVersionRequest, CompVersionResponse, ) +from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub +from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse +from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub +from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest +from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse +from .api.provenance.v2.scanoss_provenance_pb2_grpc import ProvenanceStub +from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse +from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub +from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse +from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub from .scanossbase import ScanossBase -from . import __version__ DEFAULT_URL = 'https://api.osskb.org' # default free service URL DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL SCANOSS_GRPC_URL = os.environ.get('SCANOSS_GRPC_URL') if os.environ.get('SCANOSS_GRPC_URL') else DEFAULT_URL SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else '' +MAX_CONCURRENT_REQUESTS = 5 + class ScanossGrpc(ScanossBase): """ Client for gRPC functionality """ - def __init__( + def __init__( # noqa: PLR0913 self, url: str = None, debug: bool = False, @@ -222,31 +230,54 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: :return: Server response or None """ if not dependencies: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None - request_id = str(uuid.uuid4()) - resp: DependencyResponse - try: - files_json = dependencies.get('files') - if files_json is None or len(files_json) == 0: - self.print_stderr(f'ERROR: No dependency data supplied to send to gRPC service.') + + files_json = dependencies.get('files') + + if files_json is None or len(files_json) == 0: + self.print_stderr('ERROR: No dependency data supplied to send to gRPC service.') + return None + + def process_file(file): + request_id = str(uuid.uuid4()) + try: + file_request = {'files': [file]} + + request = ParseDict(file_request, DependencyRequest()) + request.depth = depth + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) + self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...') + resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout) + + return MessageToDict(resp, preserving_proto_field_name=True) + except Exception as e: + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) return None - request = ParseDict(dependencies, DependencyRequest()) # Parse the JSON/Dict into the dependency object - request.depth = depth - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...') - resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - if not self._check_status_response(resp.status, request_id): - return None - return MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dictionary - return None + + all_responses = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS) as executor: + future_to_file = {executor.submit(process_file, file): file for file in files_json} + + for future in concurrent.futures.as_completed(future_to_file): + response = future.result() + if response: + all_responses.append(response) + + SUCCESS_STATUS = 'SUCCESS' + + merged_response = {'files': [], 'status': {'status': SUCCESS_STATUS, 'message': 'Success'}} + for response in all_responses: + if response: + if 'files' in response and len(response['files']) > 0: + merged_response['files'].append(response['files'][0]) + # Overwrite the status if the any of the responses was not successful + if 'status' in response and response['status']['status'] != SUCCESS_STATUS: + merged_response['status'] = response['status'] + return merged_response def get_crypto_json(self, purls: dict) -> dict: """ @@ -255,7 +286,7 @@ def get_crypto_json(self, purls: dict) -> dict: :return: Server response or None """ if not purls: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) resp: AlgorithmResponse @@ -285,7 +316,7 @@ def get_vulnerabilities_json(self, purls: dict) -> dict: :return: Server response or None """ if not purls: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) resp: VulnerabilityResponse @@ -315,7 +346,7 @@ def get_semgrep_json(self, purls: dict) -> dict: :return: Server response or None """ if not purls: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) resp: SemgrepResponse @@ -345,7 +376,7 @@ def search_components_json(self, search: dict) -> dict: :return: Server response or None """ if not search: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) resp: CompSearchResponse @@ -375,7 +406,7 @@ def get_component_versions_json(self, search: dict) -> dict: :return: Server response or None """ if not search: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) resp: CompVersionResponse @@ -404,6 +435,10 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st :param status_response: Status Response :return: True if successful, False otherwise """ + + SUCCEDED_WITH_WARNINGS_STATUS_CODE = 2 + FAILED_STATUS_CODE = 3 + if not status_response: self.print_stderr(f'Warning: No status response supplied (rqId: {request_id}). Assuming it was ok.') return True @@ -411,11 +446,11 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st status_code: StatusCode = status_response.status if status_code > 1: ret_val = False # default to failed - msg = "Unsuccessful" - if status_code == 2: - msg = "Succeeded with warnings" + msg = 'Unsuccessful' + if status_code == SUCCEDED_WITH_WARNINGS_STATUS_CODE: + msg = 'Succeeded with warnings' ret_val = True # No need to fail as it succeeded with warnings - elif status_code == 3: + elif status_code == FAILED_STATUS_CODE: msg = 'Failed with warnings' self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}') return ret_val @@ -428,10 +463,10 @@ def _get_proxy_config(self): :param self: """ if self.grpc_proxy: - self.print_debug(f'Setting GRPC (grpc_proxy) proxy...') + self.print_debug('Setting GRPC (grpc_proxy) proxy...') os.environ['grpc_proxy'] = self.grpc_proxy elif self.proxy: - self.print_debug(f'Setting GRPC (http_proxy/https_proxy) proxies...') + self.print_debug('Setting GRPC (http_proxy/https_proxy) proxies...') os.environ['http_proxy'] = self.proxy os.environ['https_proxy'] = self.proxy elif self.pac: @@ -450,7 +485,7 @@ def get_provenance_json(self, purls: dict) -> dict: :return: Server response or None """ if not purls: - self.print_stderr(f'ERROR: No message supplied to send to gRPC service.') + self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) resp: ProvenanceResponse @@ -461,8 +496,9 @@ def get_provenance_json(self, purls: dict) -> dict: self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...') resp = self.provenance_stub.GetComponentProvenance(request, metadata=metadata, timeout=self.timeout) except Exception as e: - self.print_stderr(f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message ' - f'(rqId: {request_id}): {e}') + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) else: if resp: if not self._check_status_response(resp.status, request_id): @@ -470,6 +506,8 @@ def get_provenance_json(self, purls: dict) -> dict: resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict return resp_dict return None + + # # End of ScanossGrpc Class # From 342ac5dbd6303a3b654032c666237735d6164ba4 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 14 Mar 2025 15:08:12 +0100 Subject: [PATCH 284/489] fix: SP-2195 update version + changelog --- CHANGELOG.md | 7 ++++++- src/scanoss/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd7f9a3f..ff593af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.6] - 2025-03-14 +### Fixed +- Fixed timeout issue with dependency scan + ## [1.20.5] - 2025-03-13 ### Added - Improved documentation on spdxlite.py file @@ -483,4 +487,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 [1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 [1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 -[1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 \ No newline at end of file +[1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 +[1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 00f19ec1..21ab20a7 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.5' +__version__ = '1.20.6' From 46f693d0a9a4dc28c31de170e1a64c13dc10af03 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 14 Mar 2025 15:12:50 +0100 Subject: [PATCH 285/489] fix: SP-2195 update version + changelog --- CHANGELOG.md | 7 ++----- src/scanoss/__init__.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff593af1..4d59d470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.20.6] - 2025-03-14 +## [1.20.5] - 2025-03-13 ### Fixed - Fixed timeout issue with dependency scan - -## [1.20.5] - 2025-03-13 ### Added - Improved documentation on spdxlite.py file ### Modified @@ -487,5 +485,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 [1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 [1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 -[1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 -[1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 \ No newline at end of file +[1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 21ab20a7..00f19ec1 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.6' +__version__ = '1.20.5' From 4dd0c739d6286a68c08037ca015b340fa98eb654 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Fri, 14 Mar 2025 10:58:49 -0300 Subject: [PATCH 286/489] feat:SP-2244 Adds generic HTTP/gRPC header options --- src/scanoss/cli.py | 44 +++++++++++++++++++++++++++++++++++++- src/scanoss/components.py | 2 ++ src/scanoss/scanner.py | 4 ++++ src/scanoss/scanossapi.py | 11 ++++++++++ src/scanoss/scanossgrpc.py | 13 +++++++++++ 5 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 54a6db0a..730b68a0 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -25,9 +25,12 @@ import argparse import os import sys +from array import array +from cgi import parse_header from pathlib import Path import pypac +from typing import List from . import __version__ from .components import Components @@ -567,6 +570,12 @@ def setup_args() -> None: # noqa: PLR0915 help='GRPC Proxy URL to use for connections (optional). ' 'Can also use the environment variable "grcp_proxy=:"', ) + p.add_argument( + '--header','-HDR', + action='append', # This allows multiple -H flags + type=str, + help='Headers to be sent on request (e.g., -H "Name: Value") - can be used multiple times' + ) # Help/Trace command options for p in [ @@ -707,6 +716,7 @@ def wfp(parser, args): strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, + req_headers = process_req_headers(args.header), ) if args.stdin: contents = sys.stdin.buffer.read() @@ -903,6 +913,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, + req_headers= process_req_headers(args.header), ) if args.wfp: if not scanner.is_file_or_snippet_scan(): @@ -1228,6 +1239,7 @@ def comp_crypto(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') sys.exit(1) pac_file = get_pac_file(args.pac) + comps = Components( debug=args.debug, trace=args.trace, @@ -1239,6 +1251,7 @@ def comp_crypto(parser, args): grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, + req_headers= process_req_headers(args.header), ) if not comps.get_crypto_details(args.input, args.purl, args.output): sys.exit(1) @@ -1273,6 +1286,7 @@ def comp_vulns(parser, args): grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, + req_headers=process_req_headers(args.header), ) if not comps.get_vulnerabilities(args.input, args.purl, args.output): sys.exit(1) @@ -1307,6 +1321,7 @@ def comp_semgrep(parser, args): grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, + req_headers=process_req_headers(args.header), ) if not comps.get_semgrep_details(args.input, args.purl, args.output): sys.exit(1) @@ -1344,6 +1359,7 @@ def comp_search(parser, args): grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, + req_headers=process_req_headers(args.header), ) if not comps.search_components( args.output, @@ -1388,6 +1404,7 @@ def comp_versions(parser, args): grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, + req_headers=process_req_headers(args.header), ) if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit): sys.exit(1) @@ -1412,7 +1429,7 @@ def comp_provenance(parser, args): pac_file = get_pac_file(args.pac) comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout) + timeout=args.timeout, req_headers=process_req_headers(args.header)) if not comps.get_provenance_details(args.input, args.purl, args.output): sys.exit(1) @@ -1456,6 +1473,31 @@ def results(parser, args): results.apply_filters().present() +def process_req_headers(headers_array: List[str]) -> dict: + """ + Process a list of header strings in the format "Name: Value" into a dictionary. + + Args: + headers_array (list): List of header strings from command line args + + Returns: + dict: Dictionary of header name-value pairs + """ + # Check if headers_array is empty + if not headers_array: + # Array is empty + return {} + + dict_headers = {} + for header_str in headers_array: + # Split each "Name: Value" header + parts = header_str.split(":", 1) + if len(parts) == 2: + name = parts[0].strip() + value = parts[1].strip() + dict_headers[name] = value + return dict_headers + def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/components.py b/src/scanoss/components.py index e98e7a80..6af15192 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -51,6 +51,7 @@ def __init__( grpc_proxy: str = None, ca_cert: str = None, pac: PACFile = None, + req_headers: dict = None, ): """ Handle all component style requests @@ -80,6 +81,7 @@ def __init__( pac=pac, grpc_proxy=grpc_proxy, timeout=timeout, + req_headers=req_headers, ) def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None) -> Optional[dict]: diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 79f81d5b..c5dd215a 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -106,6 +106,7 @@ def __init__( strip_snippet_ids=None, skip_md5_ids=None, scan_settings: 'ScanossSettings | None' = None, + req_headers: dict = None, ): """ Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning @@ -129,6 +130,7 @@ def __init__( self.skip_folders = skip_folders self.skip_size = skip_size self.skip_extensions = skip_extensions + self.req_headers = req_headers ver_details = Scanner.version_details() self.winnowing = Winnowing( @@ -156,6 +158,7 @@ def __init__( ca_cert=ca_cert, pac=pac, retry=retry, + req_headers= self.req_headers, ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) grpc_api = ScanossGrpc( @@ -169,6 +172,7 @@ def __init__( proxy=proxy, pac=pac, grpc_proxy=grpc_proxy, + req_headers=self.req_headers, ) self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace) self.nb_threads = nb_threads diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index fc5d3522..b8a7903b 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -68,6 +68,7 @@ def __init__( ca_cert: str = None, pac: PACFile = None, retry: int = 5, + req_headers: dict = None, ): """ Initialise the SCANOSS API @@ -96,7 +97,9 @@ def __init__( self.timeout = timeout if timeout > 5 else 180 self.retry_limit = retry if retry >= 0 else 5 self.ignore_cert_errors = ignore_cert_errors + self.req_headers = req_headers if req_headers else {} self.headers = {} + if ver_details: self.headers['x-scanoss-client'] = ver_details if self.api_key: @@ -104,6 +107,14 @@ def __init__( self.headers['x-api-key'] = self.api_key self.headers['User-Agent'] = f'scanoss-py/{__version__}' self.headers['user-agent'] = f'scanoss-py/{__version__}' + + if self.req_headers: # Load generic headers + for key, value in self.req_headers.items(): + if key == 'x-api-key': # Set premium URL if x-api-key header is set + if not url and not os.environ.get('SCANOSS_GRPC_URL'): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.headers.append((key, value)) + if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index a0a14abd..931529cf 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -86,6 +86,7 @@ def __init__( # noqa: PLR0913 proxy: str = None, grpc_proxy: str = None, pac: PACFile = None, + req_headers = dict ): """ @@ -113,12 +114,23 @@ def __init__( # noqa: PLR0913 self.proxy = proxy self.grpc_proxy = grpc_proxy self.pac = pac + self.req_headers = req_headers self.metadata = [] + + if self.api_key: self.metadata.append(('x-api-key', api_key)) # Set API key if we have one if ver_details: self.metadata.append(('x-scanoss-client', ver_details)) self.metadata.append(('user-agent', f'scanoss-py/{__version__}')) + + if self.req_headers: # Load generic headers + for key, value in self.req_headers.items(): + if key == 'x-api-key': # Set premium URL if x-api-key header is set + if not url and not os.environ.get('SCANOSS_GRPC_URL'): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.metadata.append((key, value)) + secure = True if self.url.startswith('https:') else False # Is it a secure connection? if self.url.startswith('http'): u = urlparse(self.url) @@ -494,6 +506,7 @@ def get_provenance_json(self, purls: dict) -> dict: metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...') + print("PROVENANCE METADATA", metadata) resp = self.provenance_stub.GetComponentProvenance(request, metadata=metadata, timeout=self.timeout) except Exception as e: self.print_stderr( From 5e2996c669741fe67bfd545a3de3e5152d5bf1d1 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Fri, 14 Mar 2025 13:58:49 -0300 Subject: [PATCH 287/489] chore:SP-2246 Adds generic header options unit tests --- src/scanoss/scanossapi.py | 21 ++++++++----- src/scanoss/scanossgrpc.py | 20 ++++++++----- tests/cli-test.py | 60 ++++++++++++++++++++++++++++++++++++++ tests/grpc-client-test.py | 11 +++++++ tests/scanossapi-test.py | 37 +++++++++++++++++++++++ 5 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 tests/cli-test.py create mode 100644 tests/scanossapi-test.py diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index b8a7903b..d588a2c0 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -107,13 +107,7 @@ def __init__( self.headers['x-api-key'] = self.api_key self.headers['User-Agent'] = f'scanoss-py/{__version__}' self.headers['user-agent'] = f'scanoss-py/{__version__}' - - if self.req_headers: # Load generic headers - for key, value in self.req_headers.items(): - if key == 'x-api-key': # Set premium URL if x-api-key header is set - if not url and not os.environ.get('SCANOSS_GRPC_URL'): - self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium - self.headers.append((key, value)) + self.load_generic_headers() if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) @@ -272,6 +266,19 @@ def set_sbom(self, sbom): self.sbom = sbom return self + def load_generic_headers(self): + """ + Adds custom headers from req_headers to the headers collection. + + If x-api-key is present and no URL is configured (directly or via + environment), sets URL to the premium endpoint (DEFAULT_URL2). + """ + if self.req_headers: # Load generic headers + for key, value in self.req_headers.items(): + if key == 'x-api-key': # Set premium URL if x-api-key header is set + if not self.url and not os.environ.get('SCANOSS_GRPC_URL'): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.headers[key] = value # # End of ScanossApi Class diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 931529cf..5cd1d3a0 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -123,13 +123,7 @@ def __init__( # noqa: PLR0913 if ver_details: self.metadata.append(('x-scanoss-client', ver_details)) self.metadata.append(('user-agent', f'scanoss-py/{__version__}')) - - if self.req_headers: # Load generic headers - for key, value in self.req_headers.items(): - if key == 'x-api-key': # Set premium URL if x-api-key header is set - if not url and not os.environ.get('SCANOSS_GRPC_URL'): - self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium - self.metadata.append((key, value)) + self.load_generic_headers() secure = True if self.url.startswith('https:') else False # Is it a secure connection? if self.url.startswith('http'): @@ -520,7 +514,19 @@ def get_provenance_json(self, purls: dict) -> dict: return resp_dict return None + def load_generic_headers(self): + """ + Adds custom headers from req_headers to metadata. + If x-api-key is present and no URL is configured (directly or via + environment), sets URL to the premium endpoint (DEFAULT_URL2). + """ + if self.req_headers: # Load generic headers + for key, value in self.req_headers.items(): + if key == 'x-api-key': # Set premium URL if x-api-key header is set + if not self.url and not os.environ.get('SCANOSS_GRPC_URL'): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.metadata.append((key, value)) # # End of ScanossGrpc Class # diff --git a/tests/cli-test.py b/tests/cli-test.py new file mode 100644 index 00000000..453fe1ad --- /dev/null +++ b/tests/cli-test.py @@ -0,0 +1,60 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import unittest +from scanoss.cli import process_req_headers + +class MyTestCase(unittest.TestCase): + + def test_header_argument_processor(self): + # Test input + headers_input = [ + 'x-api-key:12334', + 'generic-header-1:generic-header-value-1', + ' space-header : value-space-header', + 'generic-header2 generic-header-value-2' # Note: missing colon separator + ] + + # Process headers + processed_headers = process_req_headers(headers_input) + + # Expected results (as a dictionary for easier comparison) + expected_headers = { + 'x-api-key': '12334', + 'generic-header-1': 'generic-header-value-1', + 'space-header': 'value-space-header' + # Note: generic-header2 not included as it doesn't have a colon separator + } + + # Test exact dictionary equality + self.assertEqual(processed_headers, expected_headers, + f"Headers don't match expected values.\nGot: {processed_headers}\nExpected: {expected_headers}") + + # Additional tests for specific cases + self.assertIn('x-api-key', processed_headers, "Required header 'x-api-key' missing") + self.assertEqual(processed_headers['x-api-key'], '12334', "Header value for 'x-api-key' is incorrect") + + # Test that the malformed header was not included + self.assertNotIn('generic-header2', processed_headers, + "Malformed header without colon separator should not be included") \ No newline at end of file diff --git a/tests/grpc-client-test.py b/tests/grpc-client-test.py index 57dee58b..18cad31d 100644 --- a/tests/grpc-client-test.py +++ b/tests/grpc-client-test.py @@ -121,5 +121,16 @@ def test_load_purls_file(self): # Ensure the method returned None (indicating a failure) self.assertEqual(components,expected_value) + def test_grpc_generic_metadata(self): + grpc_client = ScanossGrpc(debug=True, req_headers={'x-api-key': '123455', 'generic-header': 'generic-header-value'}) + required_keys = ('x-api-key', 'user-agent', 'x-scanoss-client', 'generic-header') + valid_metadata = True + for key, value in grpc_client.metadata: + if key not in required_keys: + valid_metadata = False + self.assertTrue(valid_metadata) + + + if __name__ == '__main__': unittest.main() diff --git a/tests/scanossapi-test.py b/tests/scanossapi-test.py new file mode 100644 index 00000000..7d6d2297 --- /dev/null +++ b/tests/scanossapi-test.py @@ -0,0 +1,37 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import unittest +from scanoss.scanossapi import ScanossApi + +class MyTestCase(unittest.TestCase): + + def test_scanoss_generic_headers(self): + scanoss_api = ScanossApi(debug=True, req_headers={'x-api-key': '123455', 'generic-header': 'generic-header-value'}) + required_keys = ('x-api-key', 'User-Agent', 'user-agent', 'generic-header') + valid_headers = True + for key, value in scanoss_api.headers.items(): + if key not in required_keys: + valid_headers = False + self.assertTrue(valid_headers) \ No newline at end of file From 58be215fc8ca02e9954e6e9b9804d64d53fea37d Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Fri, 14 Mar 2025 14:30:37 -0300 Subject: [PATCH 288/489] chore:SP-2247 Updates CLIENT_HELP-md with generic header options examples --- CLIENT_HELP.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 6837fafa..d9ce45df 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -222,6 +222,16 @@ The following command scans the `src` folder writing the output to `scan-results scanoss-py scan -o scan-results.json -5 37f7cd1e657aa3c30ece35995b4c59e5 -E '.h' -Z 512 -O internal -N 'd5e54c33,b03faabe' src ``` +### Scan with custom headers +Scan with custom headers. This example scans the `src` folder and sends a custom API key header with the request: +```bash +scanoss-py scan -o scan-results.json src --HDR "x-api-key:12345" +``` +Multiple Headers: You can specify any number of custom headers by repeating the --HDR option: +```bash +scanoss-py scan src --HDR "x-api-key:12345" --HDR "custom-header:value" --HDR "another-header:another-value" +``` + ### Converting RAW results into other formats The following command provides the capability to convert the RAW scan results from a SCANOSS scan into multiple different formats, including CycloneDX, SPDX Lite, CSV, etc. For the full set of formats, please run: @@ -247,6 +257,12 @@ For the latest list of sub-commands, please run: scanoss-py comp --help ``` +All component sub-commands support custom headers using the `--HDR` option: +```bash +scanoss-py comp search "jquery" --HDR "x-api-key:12345" +scanoss-py comp vulns "jquery@3.6.0" --HDR "x-api-key:12345" --HDR "custom-header:value" + + #### Component Vulnerabilities The following command provides the capability to search the SCANOSS KB for component vulnerabilities: ```bash From 3afc595eaeeb3e5631545c2e19baa6b89a815676 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 18 Mar 2025 08:29:30 -0300 Subject: [PATCH 289/489] chore: Sets URL based on generic header --- CLIENT_HELP.md | 12 ++++++------ src/scanoss/cli.py | 4 ++-- src/scanoss/scanossapi.py | 12 ++++++++---- src/scanoss/scanossgrpc.py | 13 ++++++++----- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index d9ce45df..18098972 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -225,11 +225,11 @@ scanoss-py scan -o scan-results.json -5 37f7cd1e657aa3c30ece35995b4c59e5 -E '.h' ### Scan with custom headers Scan with custom headers. This example scans the `src` folder and sends a custom API key header with the request: ```bash -scanoss-py scan -o scan-results.json src --HDR "x-api-key:12345" +scanoss-py scan -o scan-results.json src --hdr "x-api-key:12345" ``` -Multiple Headers: You can specify any number of custom headers by repeating the --HDR option: +Multiple Headers: You can specify any number of custom headers by repeating the --hdr option: ```bash -scanoss-py scan src --HDR "x-api-key:12345" --HDR "custom-header:value" --HDR "another-header:another-value" +scanoss-py scan src --hdr "x-api-key:12345" --hdr "custom-header:value" --hdr "another-header:another-value" ``` ### Converting RAW results into other formats @@ -257,10 +257,10 @@ For the latest list of sub-commands, please run: scanoss-py comp --help ``` -All component sub-commands support custom headers using the `--HDR` option: +All component sub-commands support custom headers using the `--hdr` option: ```bash -scanoss-py comp search "jquery" --HDR "x-api-key:12345" -scanoss-py comp vulns "jquery@3.6.0" --HDR "x-api-key:12345" --HDR "custom-header:value" +scanoss-py comp search "jquery" --hdr "x-api-key:12345" +scanoss-py comp vulns "jquery@3.6.0" --hdr "x-api-key:12345" --hdr "custom-header:value" #### Component Vulnerabilities diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 730b68a0..afafa213 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -571,10 +571,10 @@ def setup_args() -> None: # noqa: PLR0915 'Can also use the environment variable "grcp_proxy=:"', ) p.add_argument( - '--header','-HDR', + '--header','-hdr', action='append', # This allows multiple -H flags type=str, - help='Headers to be sent on request (e.g., -H "Name: Value") - can be used multiple times' + help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times' ) # Help/Trace command options diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index d588a2c0..899b9e4e 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -87,10 +87,8 @@ def __init__( HTTPS_PROXY='http://:' """ super().__init__(debug, trace, quiet) - self.url = url if url else SCANOSS_SCAN_URL - self.api_key = api_key if api_key else SCANOSS_API_KEY - if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'): - self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.url = url + self.api_key = api_key self.sbom = None self.scan_format = scan_format if scan_format else 'plain' self.flags = flags @@ -109,6 +107,11 @@ def __init__( self.headers['user-agent'] = f'scanoss-py/{__version__}' self.load_generic_headers() + self.url = url if url else SCANOSS_SCAN_URL + self.api_key = api_key if api_key else SCANOSS_API_KEY + if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 @@ -278,6 +281,7 @@ def load_generic_headers(self): if key == 'x-api-key': # Set premium URL if x-api-key header is set if not self.url and not os.environ.get('SCANOSS_GRPC_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.api_key = value self.headers[key] = value # diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 5cd1d3a0..d873ff32 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -104,12 +104,8 @@ def __init__( # noqa: PLR0913 grpc_proxy='http://:' """ super().__init__(debug, trace, quiet) - self.url = url if url else SCANOSS_GRPC_URL + self.url = url self.api_key = api_key if api_key else SCANOSS_API_KEY - if self.api_key and not url and not os.environ.get('SCANOSS_GRPC_URL'): - self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium - self.url = self.url.lower() - self.orig_url = self.url # Used for proxy lookup self.timeout = timeout self.proxy = proxy self.grpc_proxy = grpc_proxy @@ -125,6 +121,12 @@ def __init__( # noqa: PLR0913 self.metadata.append(('user-agent', f'scanoss-py/{__version__}')) self.load_generic_headers() + self.url = url if url else SCANOSS_GRPC_URL + if self.api_key and not url and not os.environ.get('SCANOSS_GRPC_URL'): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.url = self.url.lower() + self.orig_url = self.url # Used for proxy lookup + secure = True if self.url.startswith('https:') else False # Is it a secure connection? if self.url.startswith('http'): u = urlparse(self.url) @@ -526,6 +528,7 @@ def load_generic_headers(self): if key == 'x-api-key': # Set premium URL if x-api-key header is set if not self.url and not os.environ.get('SCANOSS_GRPC_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.api_key = value self.metadata.append((key, value)) # # End of ScanossGrpc Class From 8aa0fd6e317cfbdad40a4dd2428466619f00f9dc Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz <166400360+ortizjeronimo@users.noreply.github.com> Date: Tue, 18 Mar 2025 16:13:38 -0300 Subject: [PATCH 290/489] corrected client_help examples --- CLIENT_HELP.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 18098972..8bb89339 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -225,11 +225,11 @@ scanoss-py scan -o scan-results.json -5 37f7cd1e657aa3c30ece35995b4c59e5 -E '.h' ### Scan with custom headers Scan with custom headers. This example scans the `src` folder and sends a custom API key header with the request: ```bash -scanoss-py scan -o scan-results.json src --hdr "x-api-key:12345" +scanoss-py scan -o scan-results.json src -hdr "x-api-key:12345" ``` -Multiple Headers: You can specify any number of custom headers by repeating the --hdr option: +Multiple Headers: You can specify any number of custom headers by repeating the -hdr option: ```bash -scanoss-py scan src --hdr "x-api-key:12345" --hdr "custom-header:value" --hdr "another-header:another-value" +scanoss-py scan src -hdr "x-api-key:12345" -hdr "custom-header:value" -hdr "another-header:another-value" ``` ### Converting RAW results into other formats @@ -257,10 +257,11 @@ For the latest list of sub-commands, please run: scanoss-py comp --help ``` -All component sub-commands support custom headers using the `--hdr` option: +All component sub-commands support custom headers using the `-hdr` option: ```bash -scanoss-py comp search "jquery" --hdr "x-api-key:12345" -scanoss-py comp vulns "jquery@3.6.0" --hdr "x-api-key:12345" --hdr "custom-header:value" +scanoss-py comp search "jquery" -hdr "x-api-key:12345" +scanoss-py comp vulns "jquery@3.6.0" -hdr "x-api-key:12345" -hdr "custom-header:value" +scanoss-py comp crypto --purl "pkg:github/madler/pigz" -header "x-api-key:12345" #### Component Vulnerabilities From 825adaba866cb7b2ea7d613e448ebc5402d02cb6 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Wed, 19 Mar 2025 04:58:05 -0300 Subject: [PATCH 291/489] chore:Fixes linter issues --- src/scanoss/scanossgrpc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index d873ff32..1ef3fcaf 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -86,7 +86,7 @@ def __init__( # noqa: PLR0913 proxy: str = None, grpc_proxy: str = None, pac: PACFile = None, - req_headers = dict + req_headers: dict = None, ): """ @@ -502,7 +502,6 @@ def get_provenance_json(self, purls: dict) -> dict: metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...') - print("PROVENANCE METADATA", metadata) resp = self.provenance_stub.GetComponentProvenance(request, metadata=metadata, timeout=self.timeout) except Exception as e: self.print_stderr( From 2541057fbab5b5f5d48ac91c04fe708fa99194ae Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Wed, 19 Mar 2025 05:00:32 -0300 Subject: [PATCH 292/489] chore: Upgrades version to v1.20.6 --- CHANGELOG.md | 7 ++++++- src/scanoss/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d59d470..edb5ac16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.20.6] - 2025-03-19 +### Added +- Added HTTP/gRPC generic headers + ## [1.20.5] - 2025-03-13 ### Fixed - Fixed timeout issue with dependency scan @@ -485,4 +489,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.2]: https://github.com/scanoss/scanoss.py/compare/v1.20.1...v1.20.2 [1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 [1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 -[1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 \ No newline at end of file +[1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 +[1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 00f19ec1..21ab20a7 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.5' +__version__ = '1.20.6' From 4a0945a51d9f92b1097bde381601341d80619248 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Wed, 19 Mar 2025 05:53:39 -0300 Subject: [PATCH 293/489] chore: Fixes HPMS test --- src/scanoss/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index afafa213..ca16b3d2 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -614,7 +614,6 @@ def setup_args() -> None: # noqa: PLR0915 sys.exit(1) args.func(parser, args) # Execute the function associated with the sub-command - def ver(*_): """ Run the "ver" sub-command @@ -716,7 +715,6 @@ def wfp(parser, args): strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, - req_headers = process_req_headers(args.header), ) if args.stdin: contents = sys.stdin.buffer.read() From 57122621d9f39a1ac63db207c8baabab40048d19 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Wed, 19 Mar 2025 06:10:00 -0300 Subject: [PATCH 294/489] chore: Fixes linter issues --- src/scanoss/cli.py | 5 ++--- src/scanoss/components.py | 6 +++--- src/scanoss/scanner.py | 41 +++++++++++++++++++------------------- src/scanoss/scanossapi.py | 6 +++--- src/scanoss/scanossgrpc.py | 2 +- tests/cli-test.py | 10 ++++++++-- tests/grpc-client-test.py | 3 ++- tests/scanossapi-test.py | 3 ++- 8 files changed, 42 insertions(+), 34 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ca16b3d2..bde476a6 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -25,8 +25,6 @@ import argparse import os import sys -from array import array -from cgi import parse_header from pathlib import Path import pypac @@ -53,6 +51,7 @@ MIN_TIMEOUT_VALUE = 5 DEFAULT_RETRY = 5 PYTHON3_OR_LATER = 3 +HEADER_PARTS_COUNT = 2 def print_stderr(*args, **kwargs): """ @@ -1490,7 +1489,7 @@ def process_req_headers(headers_array: List[str]) -> dict: for header_str in headers_array: # Split each "Name: Value" header parts = header_str.split(":", 1) - if len(parts) == 2: + if len(parts) == HEADER_PARTS_COUNT: name = parts[0].strip() value = parts[1].strip() dict_headers[name] = value diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 6af15192..fcecec56 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -25,7 +25,7 @@ import json import os import sys -from typing import TextIO, Optional, List +from typing import List, Optional, TextIO from pypac.parser import PACFile @@ -39,7 +39,7 @@ class Components(ScanossBase): Class for Component functionality """ - def __init__( + def __init__( # noqa: PLR0913, PLR0915 self, debug: bool = False, trace: bool = False, @@ -244,7 +244,7 @@ def get_semgrep_details(self, json_file: str = None, purls: [] = None, output_fi self._close_file(output_file, file) return success - def search_components( + def search_components( # noqa: PLR0913, PLR0915 self, output_file: str = None, json_file: str = None, diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index c5dd215a..4b59903e 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -69,7 +69,7 @@ class Scanner(ScanossBase): Handle the scanning of files, snippets and dependencies """ - def __init__( + def __init__( # noqa: PLR0913, PLR0915 self, wfp: str = None, scan_output: str = None, @@ -306,7 +306,7 @@ def scan_folder_with_options( success = True if not scan_dir: - raise Exception(f'ERROR: Please specify a folder to scan') + raise Exception('ERROR: Please specify a folder to scan') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): @@ -390,7 +390,8 @@ def scan_folder(self, scan_dir: str) -> bool: file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue + # If the WFP is bigger than the max post size and we already have something stored in the scan block, + # add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -440,7 +441,7 @@ def __run_scan_threaded(self, scan_started: bool, file_count: int) -> bool: self.threaded_scan.update_bar(create=True, file_count=file_count) if not scan_started: if not self.threaded_scan.run(wait=False): # Run the scan but do not wait for it to complete - self.print_stderr(f'Warning: Some errors encounted while scanning. Results might be incomplete.') + self.print_stderr('Warning: Some errors encounted while scanning. Results might be incomplete.') success = False return success @@ -461,14 +462,14 @@ def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> b dep_responses = None if self.is_file_or_snippet_scan(): if not self.threaded_scan.complete(): # Wait for the scans to complete - self.print_stderr(f'Warning: Scanning analysis ran into some trouble.') + self.print_stderr('Warning: Scanning analysis ran into some trouble.') success = False self.threaded_scan.complete_bar() scan_responses = self.threaded_scan.responses if self.is_dependency_scan(): self.print_msg('Retrieving dependency data...') if not self.threaded_deps.complete(): - self.print_stderr(f'Warning: Dependency analysis ran into some trouble.') + self.print_stderr('Warning: Dependency analysis ran into some trouble.') success = False dep_responses = self.threaded_deps.responses @@ -550,7 +551,7 @@ def scan_file_with_options( """ success = True if not file: - raise Exception(f'ERROR: Please specify a file to scan') + raise Exception('ERROR: Please specify a file to scan') if not os.path.exists(file) or not os.path.isfile(file): raise Exception(f'ERROR: Specified file does not exist or is not a file: {file}') if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): @@ -587,7 +588,7 @@ def scan_file(self, file: str) -> bool: """ success = True if not file: - raise Exception(f'ERROR: Please specify a file to scan') + raise Exception('ERROR: Please specify a file to scan') if not os.path.exists(file) or not os.path.isfile(file): raise Exception(f'ERROR: Specified files does not exist or is not a file: {file}') self.print_debug(f'Fingerprinting {file}...') @@ -612,7 +613,7 @@ def scan_files(self, files: []) -> bool: """ success = True if not files: - raise Exception(f'ERROR: Please provide a non-empty list of filenames to scan') + raise Exception('ERROR: Please provide a non-empty list of filenames to scan') file_filters = FileFilters( debug=self.debug, @@ -675,7 +676,7 @@ def scan_files(self, files: []) -> bool: scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( - f'Warning: Some errors encounted while scanning. Results might be incomplete.' + 'Warning: Some errors encounted while scanning. Results might be incomplete.' ) success = False @@ -708,12 +709,12 @@ def scan_files_with_options(self, files: [], deps_file: str = None, file_map: di """ success = True if not files: - raise Exception(f'ERROR: Please specify a list of files to scan') + raise Exception('ERROR: Please specify a list of files to scan') if not self.is_file_or_snippet_scan(): raise Exception(f'ERROR: file or snippet scan options have to be set to scan files: {files}') if self.is_dependency_scan() or deps_file: raise Exception( - f'ERROR: The dependency scan option is currently not supported when scanning a list of files' + 'ERROR: The dependency scan option is currently not supported when scanning a list of files' ) if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') @@ -735,9 +736,9 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: """ success = True if not filename: - raise Exception(f'ERROR: Please specify a filename to scan') + raise Exception('ERROR: Please specify a filename to scan') if not contents: - raise Exception(f'ERROR: Please specify a file contents to scan') + raise Exception('ERROR: Please specify a file contents to scan') self.print_debug(f'Fingerprinting {filename}...') wfp = self.winnowing.wfp_for_contents(filename, False, contents) @@ -928,7 +929,7 @@ def scan_wfp_file_threaded(self, file: str = None) -> bool: scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( - f'Warning: Some errors encounted while scanning. Results might be incomplete.' + 'Warning: Some errors uncounted while scanning. Results might be incomplete.' ) success = False # End for loop @@ -952,7 +953,7 @@ def scan_wfp(self, wfp: str) -> bool: """ success = True if not wfp: - raise Exception(f'ERROR: Please specify a WFP to scan') + raise Exception('ERROR: Please specify a WFP to scan') raw_output = '{\n' scan_resp = self.scanoss_api.scan(wfp) if scan_resp is not None: @@ -988,9 +989,9 @@ def wfp_contents(self, filename: str, contents: bytes, wfp_file: str = None): :return: """ if not filename: - raise Exception(f'ERROR: Please specify a filename to scan') + raise Exception('ERROR: Please specify a filename to scan') if not contents: - raise Exception(f'ERROR: Please specify a file contents to scan') + raise Exception('ERROR: Please specify a file contents to scan') self.print_debug(f'Fingerprinting {filename}...') wfp = self.winnowing.wfp_for_contents(filename, False, contents) @@ -1009,7 +1010,7 @@ def wfp_file(self, scan_file: str, wfp_file: str = None): Fingerprint the specified file """ if not scan_file: - raise Exception(f'ERROR: Please specify a file to fingerprint') + raise Exception('ERROR: Please specify a file to fingerprint') if not os.path.exists(scan_file) or not os.path.isfile(scan_file): raise Exception(f'ERROR: Specified file does not exist or is not a file: {scan_file}') @@ -1030,7 +1031,7 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): Fingerprint the specified folder producing fingerprints """ if not scan_dir: - raise Exception(f'ERROR: Please specify a folder to fingerprint') + raise Exception('ERROR: Please specify a folder to fingerprint') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') file_filters = FileFilters( diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 899b9e4e..de51da02 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -52,7 +52,7 @@ class ScanossApi(ScanossBase): Currently support posting scan requests to the SCANOSS streaming API """ - def __init__( + def __init__( # noqa: PLR0913, PLR0915 self, scan_format: str = None, flags: str = None, @@ -116,13 +116,13 @@ def __init__( logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 if pac and not proxy: # Setup PAC session if requested (and no proxy has been explicitly set) - self.print_debug(f'Setting up PAC session...') + self.print_debug('Setting up PAC session...') self.session = PACSession(pac=pac) else: self.session = requests.sessions.Session() self.verify = None if self.ignore_cert_errors: - self.print_debug(f'Ignoring cert errors...') + self.print_debug('Ignoring cert errors...') urllib3.disable_warnings(InsecureRequestWarning) self.verify = False self.session.verify = False diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 1ef3fcaf..77f55501 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -73,7 +73,7 @@ class ScanossGrpc(ScanossBase): Client for gRPC functionality """ - def __init__( # noqa: PLR0913 + def __init__( # noqa: PLR0913, PLR0915 self, url: str = None, debug: bool = False, diff --git a/tests/cli-test.py b/tests/cli-test.py index 453fe1ad..1b0592fe 100644 --- a/tests/cli-test.py +++ b/tests/cli-test.py @@ -23,8 +23,10 @@ """ import unittest + from scanoss.cli import process_req_headers + class MyTestCase(unittest.TestCase): def test_header_argument_processor(self): @@ -49,11 +51,15 @@ def test_header_argument_processor(self): # Test exact dictionary equality self.assertEqual(processed_headers, expected_headers, - f"Headers don't match expected values.\nGot: {processed_headers}\nExpected: {expected_headers}") + f"Headers don't match expected values.\nGot:" + f" {processed_headers}\nExpected: {expected_headers}") # Additional tests for specific cases self.assertIn('x-api-key', processed_headers, "Required header 'x-api-key' missing") - self.assertEqual(processed_headers['x-api-key'], '12334', "Header value for 'x-api-key' is incorrect") + self.assertEqual( + processed_headers['x-api-key'], + '12334', + "Header value for 'x-api-key' is incorrect") # Test that the malformed header was not included self.assertNotIn('generic-header2', processed_headers, diff --git a/tests/grpc-client-test.py b/tests/grpc-client-test.py index 18cad31d..680820f3 100644 --- a/tests/grpc-client-test.py +++ b/tests/grpc-client-test.py @@ -122,7 +122,8 @@ def test_load_purls_file(self): self.assertEqual(components,expected_value) def test_grpc_generic_metadata(self): - grpc_client = ScanossGrpc(debug=True, req_headers={'x-api-key': '123455', 'generic-header': 'generic-header-value'}) + grpc_client = ScanossGrpc(debug=True, req_headers={'x-api-key': '123455', + 'generic-header': 'generic-header-value'}) required_keys = ('x-api-key', 'user-agent', 'x-scanoss-client', 'generic-header') valid_metadata = True for key, value in grpc_client.metadata: diff --git a/tests/scanossapi-test.py b/tests/scanossapi-test.py index 7d6d2297..a9fa1db1 100644 --- a/tests/scanossapi-test.py +++ b/tests/scanossapi-test.py @@ -28,7 +28,8 @@ class MyTestCase(unittest.TestCase): def test_scanoss_generic_headers(self): - scanoss_api = ScanossApi(debug=True, req_headers={'x-api-key': '123455', 'generic-header': 'generic-header-value'}) + scanoss_api = ScanossApi(debug=True, req_headers={'x-api-key': '123455', + 'generic-header': 'generic-header-value'}) required_keys = ('x-api-key', 'User-Agent', 'user-agent', 'generic-header') valid_headers = True for key, value in scanoss_api.headers.items(): From db6e8ff5b59ce00155aa6a3304cb832202dda9bb Mon Sep 17 00:00:00 2001 From: Jeronimo Ortiz <166400360+ortizjeronimo@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:55:44 -0300 Subject: [PATCH 295/489] updated markdown files and changed env variable --- CHANGELOG.md | 2 +- CLIENT_HELP.md | 2 +- src/scanoss/scanossapi.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edb5ac16..12842f80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.20.6] - 2025-03-19 ### Added -- Added HTTP/gRPC generic headers +- Added HTTP/gRPC generic headers feature using --header flag ## [1.20.5] - 2025-03-13 ### Fixed diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 8bb89339..3dc9b31c 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -229,7 +229,7 @@ scanoss-py scan -o scan-results.json src -hdr "x-api-key:12345" ``` Multiple Headers: You can specify any number of custom headers by repeating the -hdr option: ```bash -scanoss-py scan src -hdr "x-api-key:12345" -hdr "custom-header:value" -hdr "another-header:another-value" +scanoss-py scan src -hdr "x-api-key:12345" -hdr "Authorization: Bearer " ``` ### Converting RAW results into other formats diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index de51da02..c934c95a 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -279,7 +279,7 @@ def load_generic_headers(self): if self.req_headers: # Load generic headers for key, value in self.req_headers.items(): if key == 'x-api-key': # Set premium URL if x-api-key header is set - if not self.url and not os.environ.get('SCANOSS_GRPC_URL'): + if not self.url and not os.environ.get('SCANOSS_SCAN_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.api_key = value self.headers[key] = value From 4b2e0264a1d2a52afe92eede26b3fba31c556c45 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 11 Feb 2025 10:21:01 +0100 Subject: [PATCH 296/489] feat: ES-163 init folder hashing scan config feat: ES-163 add build children logic to folder hashing scan feat: ES-163 mimic go implementation of hashing folder content feat: ES-163 create our own crc64 implementation feat: ES-163 create our own simhash implementation based on go library feat: ES-163 better error handling feat: ES-163 update changelog, client help and version. add headers feat: ES-163 add AbstractPresenter to handle results output in a centralized way feat: ES-163 update changelog feat: ES-163 fix pr comments, update lint.yml workflow feat: ES-163 add best match and treshold arguments to hfh scan command downgrade python/protobuf versions feat: apply same filters from go minr feat: create standalone presenters that implement abstractpresenter feat: add progress tracking while scanning, clean folder-scan command arguments feat: use progress instead of rich feat: add folder hash sub-command, fix hfh spinner feat: update docs feat: update default files/directories to skip while scanning feat: add extra checks in file filters for folder hashing scan feat: comment new skipped dirs for now feat: align with go client again feat: filter hidden file/folders feat: add specific filters for hfh feat: add missing command args feat: ES-163 add build children logic to folder hashing scan feat: ES-163 mimic go implementation of hashing folder content feat: ES-163 create our own crc64 implementation feat: ES-163 create our own simhash implementation based on go library feat: ES-163 better error handling feat: ES-163 update changelog, client help and version. add headers feat: ES-163 add AbstractPresenter to handle results output in a centralized way feat: ES-163 update changelog feat: ES-163 fix pr comments, update lint.yml workflow feat: update changelog --- .github/workflows/lint.yml | 10 +- CHANGELOG.md | 10 +- CLIENT_HELP.md | 9 + docs/source/index.rst | 73 ++++ pyproject.toml | 7 +- requirements.txt | 3 +- setup.cfg | 1 + .../options/annotations_pb2.py | 21 +- .../options/annotations_pb2_grpc.py | 2 +- .../options/openapiv2_pb2.py | 194 +++++------ .../options/openapiv2_pb2_grpc.py | 2 +- src/scanoss/__init__.py | 2 +- .../api/common/v2/scanoss_common_pb2.py | 36 +- .../api/common/v2/scanoss_common_pb2_grpc.py | 2 +- .../components/v2/scanoss_components_pb2.py | 86 ++--- .../v2/scanoss_components_pb2_grpc.py | 238 +++++-------- .../v2/scanoss_cryptography_pb2.py | 64 ++-- .../v2/scanoss_cryptography_pb2_grpc.py | 260 ++++++++++---- .../v2/scanoss_dependencies_pb2.py | 54 ++- .../v2/scanoss_dependencies_pb2_grpc.py | 124 +++---- .../provenance/v2/scanoss_provenance_pb2.py | 41 ++- .../v2/scanoss_provenance_pb2_grpc.py | 2 +- .../api/scanning/v2/scanoss_scanning_pb2.py | 30 +- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 110 +++--- .../api/semgrep/v2/scanoss_semgrep_pb2.py | 40 +-- .../semgrep/v2/scanoss_semgrep_pb2_grpc.py | 120 +++---- .../v2/scanoss_vulnerabilities_pb2.py | 64 ++-- .../v2/scanoss_vulnerabilities_pb2_grpc.py | 181 ++++------ src/scanoss/cli.py | 268 +++++++++++--- src/scanoss/constants.py | 12 + src/scanoss/file_filters.py | 329 +++++++++++++++--- src/scanoss/results.py | 189 +++++----- src/scanoss/scanners/__init__.py | 23 ++ src/scanoss/scanners/folder_hasher.py | 290 +++++++++++++++ src/scanoss/scanners/scanner_config.py | 73 ++++ src/scanoss/scanners/scanner_hfh.py | 160 +++++++++ src/scanoss/scanossbase.py | 6 +- src/scanoss/scanossgrpc.py | 125 ++++++- src/scanoss/utils/abstract_presenter.py | 68 ++++ src/scanoss/utils/crc64.py | 96 +++++ src/scanoss/utils/simhash.py | 198 +++++++++++ version.py | 5 +- 42 files changed, 2565 insertions(+), 1063 deletions(-) create mode 100644 src/scanoss/constants.py create mode 100644 src/scanoss/scanners/__init__.py create mode 100644 src/scanoss/scanners/folder_hasher.py create mode 100644 src/scanoss/scanners/scanner_config.py create mode 100644 src/scanoss/scanners/scanner_hfh.py create mode 100644 src/scanoss/utils/abstract_presenter.py create mode 100644 src/scanoss/utils/crc64.py create mode 100644 src/scanoss/utils/simhash.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a110d651..3925cc90 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,12 +32,17 @@ jobs: # List all changed Python files since the merge base. files=$(git diff --name-only "$merge_base" HEAD | grep '\.py$' || true) + # Filter out files that match exclude patterns from pyproject.toml + # this is a temporary workaround until we fix all the lint errors + filtered_files=$(echo "$files" | grep -v -E 'tests/|test_.*\.py|src/protoc_gen_swagger/|src/scanoss/api/' || true) + # Use the multi-line syntax for outputs. echo "files<> "$GITHUB_OUTPUT" - echo "${files}" >> "$GITHUB_OUTPUT" + echo "${filtered_files}" >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" - echo "Changed files: ${files}" + echo "Changed files before filtering: ${files}" + echo "Changed files after filtering: ${filtered_files}" - name: Run Ruff on changed files run: | @@ -50,3 +55,4 @@ jobs: # Pass the list of changed files to Ruff. echo "${{ steps.changed_files.outputs.files }}" | xargs ruff check fi + diff --git a/CHANGELOG.md b/CHANGELOG.md index 12842f80..5c9fbdae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.21.0] - 2025-03-27 +### Added +- Add folder-scan subcommand +- Add folder-hash subcommand +- Add AbstractPresenter class for presenting output in a given format +- Add several reusable helper functions for constructing config objects from CLI args + ## [1.20.6] - 2025-03-19 ### Added - Added HTTP/gRPC generic headers feature using --header flag @@ -490,4 +497,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.3]: https://github.com/scanoss/scanoss.py/compare/v1.20.2...v1.20.3 [1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 [1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 -[1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 \ No newline at end of file +[1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 +[1.21.0]: https://github.com/scanoss/scanoss.py/compare/v1.20.6...v1.21.0 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 3dc9b31c..8d37fbe8 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -420,4 +420,13 @@ scanoss-py insp undeclared -i scan-results.json --output undeclared-summary.jira The following command can be used to inspect for undeclared components and save the results in Jira Markdown format. ```bash scanoss-py insp copyleft -i scan-results.json --output copyleft-summary.jiramd --status copyleft-status.jiramd --format jira_md +``` + +### Folder-Scan a Project Folder + +The new `folder-scan` subcommand performs a comprehensive scan on an entire directory by recursively processing files to generate folder-level fingerprints. It computes CRC64 hashes and simhash values to detect directory-level similarities, which is especially useful for comparing large code bases or detecting duplicate folder structures. + +**Usage:** +```shell +scanoss-py folder-scan /path/to/folder -o folder-scan-results.json ``` \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 3d855bd1..cfc073c7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -228,6 +228,79 @@ Convert file format to plain, SPDX-Lite, CycloneDX or csv. * - --format , -f - Indicates the result output format: {plain, cyclonedx, spdxlite, csv}. (optional - default plain) +-------------------------------- +Folder Scanning: folder-scan, fs +-------------------------------- + +Performs a comprehensive scan of a directory using folder hashing to identify components and their matches. + +.. code-block:: bash + + scanoss-py folder-scan + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --output , -o + - Output result file name (optional - default STDOUT) + * - --format , -f + - Output format: {json} (optional - default json) + * - --timeout , -M + - Timeout in seconds for API communication (optional - default 600) + * - --best-match, -bm + - Enable best match mode (optional - default: False) + * - --threshold <1-100> + - Threshold for result matching (optional - default: 100) + * - --settings , -st + - Settings file to use for scanning (optional - default scanoss.json) + * - --skip-settings-file, -stf + - Skip default settings file (scanoss.json) if it exists + * - --key , -k + - SCANOSS API Key token (optional - not required for default OSSKB URL) + * - --proxy + - Proxy URL to use for connections + * - --pac + - Proxy auto configuration. Specify a file, http url or "auto" + * - --ca-cert + - Alternative certificate PEM file + * - --api2url + - SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org) + * - --grpc-proxy + - GRPC Proxy URL to use for connections + +-------------------------------- +Folder Hashing: folder-hash, fh +-------------------------------- + +Generates cryptographic hashes for files in a given directory and its subdirectories. + +.. code-block:: bash + + scanoss-py folder-hash + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --output , -o + - Output result file name (optional - default STDOUT) + * - --format , -f + - Output format: {json} (optional - default json) + * - --settings , -st + - Settings file to use for scanning (optional - default scanoss.json) + * - --skip-settings-file, -stf + - Skip default settings file (scanoss.json) if it exists + +Both commands also support these general options: + * --debug, -d: Enable debug messages + * --trace, -t: Enable trace messages + * --quiet, -q: Enable quiet mode + ----------------- Component: ----------------- diff --git a/pyproject.toml b/pyproject.toml index ef531e04..d740f7c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,12 @@ select = ["E", "F", "I", "PL"] line-length = 120 # Assume Python 3.7+ target-version = "py37" -exclude = ["tests/*", "test_*.py", "src/scanoss/cli.py"] +exclude = [ + "tests/*", + "test_*.py", + "src/protoc_gen_swagger/*", + "src/scanoss/api/*", +] [tool.ruff.format] quote-style = "single" diff --git a/requirements.txt b/requirements.txt index 0b464ec4..70db9fa6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ google-api-core importlib_resources packageurl-python pathspec -jsonschema \ No newline at end of file +jsonschema +crc \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 086a1fde..5880873e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,7 @@ install_requires = packageurl-python pathspec jsonschema + crc [options.extras_require] diff --git a/src/protoc_gen_swagger/options/annotations_pb2.py b/src/protoc_gen_swagger/options/annotations_pb2.py index 58f85f85..c568f388 100644 --- a/src/protoc_gen_swagger/options/annotations_pb2.py +++ b/src/protoc_gen_swagger/options/annotations_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: protoc-gen-swagger/options/annotations.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,19 +15,17 @@ from protoc_gen_swagger.options import openapiv2_pb2 as protoc__gen__swagger_dot_options_dot_openapiv2__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b"\n,protoc-gen-swagger/options/annotations.proto\x12'grpc.gateway.protoc_gen_swagger.options\x1a google/protobuf/descriptor.proto\x1a*protoc-gen-swagger/options/openapiv2.proto:j\n\x11openapiv2_swagger\x12\x1c.google.protobuf.FileOptions\x18\x92\x08 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Swagger:p\n\x13openapiv2_operation\x12\x1e.google.protobuf.MethodOptions\x18\x92\x08 \x01(\x0b\x32\x32.grpc.gateway.protoc_gen_swagger.options.Operation:k\n\x10openapiv2_schema\x12\x1f.google.protobuf.MessageOptions\x18\x92\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema:e\n\ropenapiv2_tag\x12\x1f.google.protobuf.ServiceOptions\x18\x92\x08 \x01(\x0b\x32,.grpc.gateway.protoc_gen_swagger.options.Tag:l\n\x0fopenapiv2_field\x12\x1d.google.protobuf.FieldOptions\x18\x92\x08 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchemaBCZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3" -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,protoc-gen-swagger/options/annotations.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a google/protobuf/descriptor.proto\x1a*protoc-gen-swagger/options/openapiv2.proto:j\n\x11openapiv2_swagger\x12\x1c.google.protobuf.FileOptions\x18\x92\x08 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Swagger:p\n\x13openapiv2_operation\x12\x1e.google.protobuf.MethodOptions\x18\x92\x08 \x01(\x0b\x32\x32.grpc.gateway.protoc_gen_swagger.options.Operation:k\n\x10openapiv2_schema\x12\x1f.google.protobuf.MessageOptions\x18\x92\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema:e\n\ropenapiv2_tag\x12\x1f.google.protobuf.ServiceOptions\x18\x92\x08 \x01(\x0b\x32,.grpc.gateway.protoc_gen_swagger.options.Tag:l\n\x0fopenapiv2_field\x12\x1d.google.protobuf.FieldOptions\x18\x92\x08 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchemaBCZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.annotations_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - google_dot_protobuf_dot_descriptor__pb2.FileOptions.RegisterExtension(openapiv2_swagger) - google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(openapiv2_operation) - google_dot_protobuf_dot_descriptor__pb2.MessageOptions.RegisterExtension(openapiv2_schema) - google_dot_protobuf_dot_descriptor__pb2.ServiceOptions.RegisterExtension(openapiv2_tag) - google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(openapiv2_field) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' + google_dot_protobuf_dot_descriptor__pb2.FileOptions.RegisterExtension(openapiv2_swagger) + google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(openapiv2_operation) + google_dot_protobuf_dot_descriptor__pb2.MessageOptions.RegisterExtension(openapiv2_schema) + google_dot_protobuf_dot_descriptor__pb2.ServiceOptions.RegisterExtension(openapiv2_tag) + google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(openapiv2_field) + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' # @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/annotations_pb2_grpc.py b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py index bf947056..2daafffe 100644 --- a/src/protoc_gen_swagger/options/annotations_pb2_grpc.py +++ b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py @@ -1,4 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc + diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2.py b/src/protoc_gen_swagger/options/openapiv2_pb2.py index 5c01c38b..0df96e43 100644 --- a/src/protoc_gen_swagger/options/openapiv2_pb2.py +++ b/src/protoc_gen_swagger/options/openapiv2_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: protoc-gen-swagger/options/openapiv2.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,105 +15,104 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n*protoc-gen-swagger/options/openapiv2.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto"\xa0\x07\n\x07Swagger\x12\x0f\n\x07swagger\x18\x01 \x01(\t\x12;\n\x04info\x18\x02 \x01(\x0b\x32-.grpc.gateway.protoc_gen_swagger.options.Info\x12\x0c\n\x04host\x18\x03 \x01(\t\x12\x11\n\tbase_path\x18\x04 \x01(\t\x12O\n\x07schemes\x18\x05 \x03(\x0e\x32>.grpc.gateway.protoc_gen_swagger.options.Swagger.SwaggerScheme\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12R\n\tresponses\x18\n \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Swagger.ResponsesEntry\x12Z\n\x14security_definitions\x18\x0b \x01(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12U\n\rexternal_docs\x18\x0e \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12T\n\nextensions\x18\x0f \x03(\x0b\x32@.grpc.gateway.protoc_gen_swagger.options.Swagger.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"B\n\rSwaggerScheme\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04HTTP\x10\x01\x12\t\n\x05HTTPS\x10\x02\x12\x06\n\x02WS\x10\x03\x12\x07\n\x03WSS\x10\x04J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\r\x10\x0e"\xa9\x05\n\tOperation\x12\x0c\n\x04tags\x18\x01 \x03(\t\x12\x0f\n\x07summary\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12U\n\rexternal_docs\x18\x04 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12\x14\n\x0coperation_id\x18\x05 \x01(\t\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12T\n\tresponses\x18\t \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Operation.ResponsesEntry\x12\x0f\n\x07schemes\x18\n \x03(\t\x12\x12\n\ndeprecated\x18\x0b \x01(\x08\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12V\n\nextensions\x18\r \x03(\x0b\x32\x42.grpc.gateway.protoc_gen_swagger.options.Operation.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01J\x04\x08\x08\x10\t"\xab\x01\n\x06Header\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x06 \x01(\t\x12\x0f\n\x07pattern\x18\r \x01(\tJ\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\n\x10\x0bJ\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\rJ\x04\x08\x0e\x10\x0fJ\x04\x08\x0f\x10\x10J\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13"\xb8\x04\n\x08Response\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12?\n\x06schema\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema\x12O\n\x07headers\x18\x03 \x03(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.Response.HeadersEntry\x12Q\n\x08\x65xamples\x18\x04 \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Response.ExamplesEntry\x12U\n\nextensions\x18\x05 \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Response.ExtensionsEntry\x1a_\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Header:\x02\x38\x01\x1a/\n\rExamplesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"\xf9\x02\n\x04Info\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x18\n\x10terms_of_service\x18\x03 \x01(\t\x12\x41\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Contact\x12\x41\n\x07license\x18\x05 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.License\x12\x0f\n\x07version\x18\x06 \x01(\t\x12Q\n\nextensions\x18\x07 \x03(\x0b\x32=.grpc.gateway.protoc_gen_swagger.options.Info.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"3\n\x07\x43ontact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\r\n\x05\x65mail\x18\x03 \x01(\t"$\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t"9\n\x15\x45xternalDocumentation\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t"\x9c\x02\n\x06Schema\x12H\n\x0bjson_schema\x18\x01 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchema\x12\x15\n\rdiscriminator\x18\x02 \x01(\t\x12\x11\n\tread_only\x18\x03 \x01(\x08\x12U\n\rexternal_docs\x18\x05 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12)\n\x07\x65xample\x18\x06 \x01(\x0b\x32\x14.google.protobuf.AnyB\x02\x18\x01\x12\x16\n\x0e\x65xample_string\x18\x07 \x01(\tJ\x04\x08\x04\x10\x05"\xe3\x05\n\nJSONSchema\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\t\x12\x11\n\tread_only\x18\x08 \x01(\x08\x12\x0f\n\x07\x65xample\x18\t \x01(\t\x12\x13\n\x0bmultiple_of\x18\n \x01(\x01\x12\x0f\n\x07maximum\x18\x0b \x01(\x01\x12\x19\n\x11\x65xclusive_maximum\x18\x0c \x01(\x08\x12\x0f\n\x07minimum\x18\r \x01(\x01\x12\x19\n\x11\x65xclusive_minimum\x18\x0e \x01(\x08\x12\x12\n\nmax_length\x18\x0f \x01(\x04\x12\x12\n\nmin_length\x18\x10 \x01(\x04\x12\x0f\n\x07pattern\x18\x11 \x01(\t\x12\x11\n\tmax_items\x18\x14 \x01(\x04\x12\x11\n\tmin_items\x18\x15 \x01(\x04\x12\x14\n\x0cunique_items\x18\x16 \x01(\x08\x12\x16\n\x0emax_properties\x18\x18 \x01(\x04\x12\x16\n\x0emin_properties\x18\x19 \x01(\x04\x12\x10\n\x08required\x18\x1a \x03(\t\x12\r\n\x05\x61rray\x18" \x03(\t\x12W\n\x04type\x18# \x03(\x0e\x32I.grpc.gateway.protoc_gen_swagger.options.JSONSchema.JSONSchemaSimpleTypes\x12\x0e\n\x06\x66ormat\x18$ \x01(\t\x12\x0c\n\x04\x65num\x18. \x03(\t"w\n\x15JSONSchemaSimpleTypes\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41RRAY\x10\x01\x12\x0b\n\x07\x42OOLEAN\x10\x02\x12\x0b\n\x07INTEGER\x10\x03\x12\x08\n\x04NULL\x10\x04\x12\n\n\x06NUMBER\x10\x05\x12\n\n\x06OBJECT\x10\x06\x12\n\n\x06STRING\x10\x07J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13J\x04\x08\x13\x10\x14J\x04\x08\x17\x10\x18J\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1dJ\x04\x08\x1d\x10\x1eJ\x04\x08\x1e\x10"J\x04\x08%\x10*J\x04\x08*\x10+J\x04\x08+\x10."w\n\x03Tag\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12U\n\rexternal_docs\x18\x03 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentationJ\x04\x08\x01\x10\x02"\xdd\x01\n\x13SecurityDefinitions\x12\\\n\x08security\x18\x01 \x03(\x0b\x32J.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions.SecurityEntry\x1ah\n\rSecurityEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.grpc.gateway.protoc_gen_swagger.options.SecurityScheme:\x02\x38\x01"\x96\x06\n\x0eSecurityScheme\x12J\n\x04type\x18\x01 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Type\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x46\n\x02in\x18\x04 \x01(\x0e\x32:.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.In\x12J\n\x04\x66low\x18\x05 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Flow\x12\x19\n\x11\x61uthorization_url\x18\x06 \x01(\t\x12\x11\n\ttoken_url\x18\x07 \x01(\t\x12?\n\x06scopes\x18\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Scopes\x12[\n\nextensions\x18\t \x03(\x0b\x32G.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01"K\n\x04Type\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\x0e\n\nTYPE_BASIC\x10\x01\x12\x10\n\x0cTYPE_API_KEY\x10\x02\x12\x0f\n\x0bTYPE_OAUTH2\x10\x03"1\n\x02In\x12\x0e\n\nIN_INVALID\x10\x00\x12\x0c\n\x08IN_QUERY\x10\x01\x12\r\n\tIN_HEADER\x10\x02"j\n\x04\x46low\x12\x10\n\x0c\x46LOW_INVALID\x10\x00\x12\x11\n\rFLOW_IMPLICIT\x10\x01\x12\x11\n\rFLOW_PASSWORD\x10\x02\x12\x14\n\x10\x46LOW_APPLICATION\x10\x03\x12\x14\n\x10\x46LOW_ACCESS_CODE\x10\x04"\xc9\x02\n\x13SecurityRequirement\x12s\n\x14security_requirement\x18\x01 \x03(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementEntry\x1a)\n\x18SecurityRequirementValue\x12\r\n\x05scope\x18\x01 \x03(\t\x1a\x91\x01\n\x18SecurityRequirementEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x64\n\x05value\x18\x02 \x01(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementValue:\x02\x38\x01"\x81\x01\n\x06Scopes\x12I\n\x05scope\x18\x01 \x03(\x0b\x32:.grpc.gateway.protoc_gen_swagger.options.Scopes.ScopeEntry\x1a,\n\nScopeEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x43ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3' -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*protoc-gen-swagger/options/openapiv2.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xa0\x07\n\x07Swagger\x12\x0f\n\x07swagger\x18\x01 \x01(\t\x12;\n\x04info\x18\x02 \x01(\x0b\x32-.grpc.gateway.protoc_gen_swagger.options.Info\x12\x0c\n\x04host\x18\x03 \x01(\t\x12\x11\n\tbase_path\x18\x04 \x01(\t\x12O\n\x07schemes\x18\x05 \x03(\x0e\x32>.grpc.gateway.protoc_gen_swagger.options.Swagger.SwaggerScheme\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12R\n\tresponses\x18\n \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Swagger.ResponsesEntry\x12Z\n\x14security_definitions\x18\x0b \x01(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12U\n\rexternal_docs\x18\x0e \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12T\n\nextensions\x18\x0f \x03(\x0b\x32@.grpc.gateway.protoc_gen_swagger.options.Swagger.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"B\n\rSwaggerScheme\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04HTTP\x10\x01\x12\t\n\x05HTTPS\x10\x02\x12\x06\n\x02WS\x10\x03\x12\x07\n\x03WSS\x10\x04J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\r\x10\x0e\"\xa9\x05\n\tOperation\x12\x0c\n\x04tags\x18\x01 \x03(\t\x12\x0f\n\x07summary\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12U\n\rexternal_docs\x18\x04 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12\x14\n\x0coperation_id\x18\x05 \x01(\t\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12T\n\tresponses\x18\t \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Operation.ResponsesEntry\x12\x0f\n\x07schemes\x18\n \x03(\t\x12\x12\n\ndeprecated\x18\x0b \x01(\x08\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12V\n\nextensions\x18\r \x03(\x0b\x32\x42.grpc.gateway.protoc_gen_swagger.options.Operation.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01J\x04\x08\x08\x10\t\"\xab\x01\n\x06Header\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x06 \x01(\t\x12\x0f\n\x07pattern\x18\r \x01(\tJ\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\n\x10\x0bJ\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\rJ\x04\x08\x0e\x10\x0fJ\x04\x08\x0f\x10\x10J\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13\"\xb8\x04\n\x08Response\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12?\n\x06schema\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema\x12O\n\x07headers\x18\x03 \x03(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.Response.HeadersEntry\x12Q\n\x08\x65xamples\x18\x04 \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Response.ExamplesEntry\x12U\n\nextensions\x18\x05 \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Response.ExtensionsEntry\x1a_\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Header:\x02\x38\x01\x1a/\n\rExamplesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"\xf9\x02\n\x04Info\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x18\n\x10terms_of_service\x18\x03 \x01(\t\x12\x41\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Contact\x12\x41\n\x07license\x18\x05 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.License\x12\x0f\n\x07version\x18\x06 \x01(\t\x12Q\n\nextensions\x18\x07 \x03(\x0b\x32=.grpc.gateway.protoc_gen_swagger.options.Info.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"3\n\x07\x43ontact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\r\n\x05\x65mail\x18\x03 \x01(\t\"$\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"9\n\x15\x45xternalDocumentation\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"\x9c\x02\n\x06Schema\x12H\n\x0bjson_schema\x18\x01 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchema\x12\x15\n\rdiscriminator\x18\x02 \x01(\t\x12\x11\n\tread_only\x18\x03 \x01(\x08\x12U\n\rexternal_docs\x18\x05 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12)\n\x07\x65xample\x18\x06 \x01(\x0b\x32\x14.google.protobuf.AnyB\x02\x18\x01\x12\x16\n\x0e\x65xample_string\x18\x07 \x01(\tJ\x04\x08\x04\x10\x05\"\xe3\x05\n\nJSONSchema\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\t\x12\x11\n\tread_only\x18\x08 \x01(\x08\x12\x0f\n\x07\x65xample\x18\t \x01(\t\x12\x13\n\x0bmultiple_of\x18\n \x01(\x01\x12\x0f\n\x07maximum\x18\x0b \x01(\x01\x12\x19\n\x11\x65xclusive_maximum\x18\x0c \x01(\x08\x12\x0f\n\x07minimum\x18\r \x01(\x01\x12\x19\n\x11\x65xclusive_minimum\x18\x0e \x01(\x08\x12\x12\n\nmax_length\x18\x0f \x01(\x04\x12\x12\n\nmin_length\x18\x10 \x01(\x04\x12\x0f\n\x07pattern\x18\x11 \x01(\t\x12\x11\n\tmax_items\x18\x14 \x01(\x04\x12\x11\n\tmin_items\x18\x15 \x01(\x04\x12\x14\n\x0cunique_items\x18\x16 \x01(\x08\x12\x16\n\x0emax_properties\x18\x18 \x01(\x04\x12\x16\n\x0emin_properties\x18\x19 \x01(\x04\x12\x10\n\x08required\x18\x1a \x03(\t\x12\r\n\x05\x61rray\x18\" \x03(\t\x12W\n\x04type\x18# \x03(\x0e\x32I.grpc.gateway.protoc_gen_swagger.options.JSONSchema.JSONSchemaSimpleTypes\x12\x0e\n\x06\x66ormat\x18$ \x01(\t\x12\x0c\n\x04\x65num\x18. \x03(\t\"w\n\x15JSONSchemaSimpleTypes\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41RRAY\x10\x01\x12\x0b\n\x07\x42OOLEAN\x10\x02\x12\x0b\n\x07INTEGER\x10\x03\x12\x08\n\x04NULL\x10\x04\x12\n\n\x06NUMBER\x10\x05\x12\n\n\x06OBJECT\x10\x06\x12\n\n\x06STRING\x10\x07J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13J\x04\x08\x13\x10\x14J\x04\x08\x17\x10\x18J\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1dJ\x04\x08\x1d\x10\x1eJ\x04\x08\x1e\x10\"J\x04\x08%\x10*J\x04\x08*\x10+J\x04\x08+\x10.\"w\n\x03Tag\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12U\n\rexternal_docs\x18\x03 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentationJ\x04\x08\x01\x10\x02\"\xdd\x01\n\x13SecurityDefinitions\x12\\\n\x08security\x18\x01 \x03(\x0b\x32J.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions.SecurityEntry\x1ah\n\rSecurityEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.grpc.gateway.protoc_gen_swagger.options.SecurityScheme:\x02\x38\x01\"\x96\x06\n\x0eSecurityScheme\x12J\n\x04type\x18\x01 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Type\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x46\n\x02in\x18\x04 \x01(\x0e\x32:.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.In\x12J\n\x04\x66low\x18\x05 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Flow\x12\x19\n\x11\x61uthorization_url\x18\x06 \x01(\t\x12\x11\n\ttoken_url\x18\x07 \x01(\t\x12?\n\x06scopes\x18\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Scopes\x12[\n\nextensions\x18\t \x03(\x0b\x32G.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"K\n\x04Type\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\x0e\n\nTYPE_BASIC\x10\x01\x12\x10\n\x0cTYPE_API_KEY\x10\x02\x12\x0f\n\x0bTYPE_OAUTH2\x10\x03\"1\n\x02In\x12\x0e\n\nIN_INVALID\x10\x00\x12\x0c\n\x08IN_QUERY\x10\x01\x12\r\n\tIN_HEADER\x10\x02\"j\n\x04\x46low\x12\x10\n\x0c\x46LOW_INVALID\x10\x00\x12\x11\n\rFLOW_IMPLICIT\x10\x01\x12\x11\n\rFLOW_PASSWORD\x10\x02\x12\x14\n\x10\x46LOW_APPLICATION\x10\x03\x12\x14\n\x10\x46LOW_ACCESS_CODE\x10\x04\"\xc9\x02\n\x13SecurityRequirement\x12s\n\x14security_requirement\x18\x01 \x03(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementEntry\x1a)\n\x18SecurityRequirementValue\x12\r\n\x05scope\x18\x01 \x03(\t\x1a\x91\x01\n\x18SecurityRequirementEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x64\n\x05value\x18\x02 \x01(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementValue:\x02\x38\x01\"\x81\x01\n\x06Scopes\x12I\n\x05scope\x18\x01 \x03(\x0b\x32:.grpc.gateway.protoc_gen_swagger.options.Scopes.ScopeEntry\x1a,\n\nScopeEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x43ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.openapiv2_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' - _SWAGGER_RESPONSESENTRY._options = None - _SWAGGER_RESPONSESENTRY._serialized_options = b'8\001' - _SWAGGER_EXTENSIONSENTRY._options = None - _SWAGGER_EXTENSIONSENTRY._serialized_options = b'8\001' - _OPERATION_RESPONSESENTRY._options = None - _OPERATION_RESPONSESENTRY._serialized_options = b'8\001' - _OPERATION_EXTENSIONSENTRY._options = None - _OPERATION_EXTENSIONSENTRY._serialized_options = b'8\001' - _RESPONSE_HEADERSENTRY._options = None - _RESPONSE_HEADERSENTRY._serialized_options = b'8\001' - _RESPONSE_EXAMPLESENTRY._options = None - _RESPONSE_EXAMPLESENTRY._serialized_options = b'8\001' - _RESPONSE_EXTENSIONSENTRY._options = None - _RESPONSE_EXTENSIONSENTRY._serialized_options = b'8\001' - _INFO_EXTENSIONSENTRY._options = None - _INFO_EXTENSIONSENTRY._serialized_options = b'8\001' - _SCHEMA.fields_by_name['example']._options = None - _SCHEMA.fields_by_name['example']._serialized_options = b'\030\001' - _SECURITYDEFINITIONS_SECURITYENTRY._options = None - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_options = b'8\001' - _SECURITYSCHEME_EXTENSIONSENTRY._options = None - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_options = b'8\001' - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._options = None - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_options = b'8\001' - _SCOPES_SCOPEENTRY._options = None - _SCOPES_SCOPEENTRY._serialized_options = b'8\001' - _SWAGGER._serialized_start = 145 - _SWAGGER._serialized_end = 1073 - _SWAGGER_RESPONSESENTRY._serialized_start = 813 - _SWAGGER_RESPONSESENTRY._serialized_end = 912 - _SWAGGER_EXTENSIONSENTRY._serialized_start = 914 - _SWAGGER_EXTENSIONSENTRY._serialized_end = 987 - _SWAGGER_SWAGGERSCHEME._serialized_start = 989 - _SWAGGER_SWAGGERSCHEME._serialized_end = 1055 - _OPERATION._serialized_start = 1076 - _OPERATION._serialized_end = 1757 - _OPERATION_RESPONSESENTRY._serialized_start = 813 - _OPERATION_RESPONSESENTRY._serialized_end = 912 - _OPERATION_EXTENSIONSENTRY._serialized_start = 914 - _OPERATION_EXTENSIONSENTRY._serialized_end = 987 - _HEADER._serialized_start = 1760 - _HEADER._serialized_end = 1931 - _RESPONSE._serialized_start = 1934 - _RESPONSE._serialized_end = 2502 - _RESPONSE_HEADERSENTRY._serialized_start = 2283 - _RESPONSE_HEADERSENTRY._serialized_end = 2378 - _RESPONSE_EXAMPLESENTRY._serialized_start = 2380 - _RESPONSE_EXAMPLESENTRY._serialized_end = 2427 - _RESPONSE_EXTENSIONSENTRY._serialized_start = 914 - _RESPONSE_EXTENSIONSENTRY._serialized_end = 987 - _INFO._serialized_start = 2505 - _INFO._serialized_end = 2882 - _INFO_EXTENSIONSENTRY._serialized_start = 914 - _INFO_EXTENSIONSENTRY._serialized_end = 987 - _CONTACT._serialized_start = 2884 - _CONTACT._serialized_end = 2935 - _LICENSE._serialized_start = 2937 - _LICENSE._serialized_end = 2973 - _EXTERNALDOCUMENTATION._serialized_start = 2975 - _EXTERNALDOCUMENTATION._serialized_end = 3032 - _SCHEMA._serialized_start = 3035 - _SCHEMA._serialized_end = 3319 - _JSONSCHEMA._serialized_start = 3322 - _JSONSCHEMA._serialized_end = 4061 - _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_start = 3864 - _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_end = 3983 - _TAG._serialized_start = 4063 - _TAG._serialized_end = 4182 - _SECURITYDEFINITIONS._serialized_start = 4185 - _SECURITYDEFINITIONS._serialized_end = 4406 - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_start = 4302 - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_end = 4406 - _SECURITYSCHEME._serialized_start = 4409 - _SECURITYSCHEME._serialized_end = 5199 - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_start = 914 - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_end = 987 - _SECURITYSCHEME_TYPE._serialized_start = 4965 - _SECURITYSCHEME_TYPE._serialized_end = 5040 - _SECURITYSCHEME_IN._serialized_start = 5042 - _SECURITYSCHEME_IN._serialized_end = 5091 - _SECURITYSCHEME_FLOW._serialized_start = 5093 - _SECURITYSCHEME_FLOW._serialized_end = 5199 - _SECURITYREQUIREMENT._serialized_start = 5202 - _SECURITYREQUIREMENT._serialized_end = 5531 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_start = 5342 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_end = 5383 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_start = 5386 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_end = 5531 - _SCOPES._serialized_start = 5534 - _SCOPES._serialized_end = 5663 - _SCOPES_SCOPEENTRY._serialized_start = 5619 - _SCOPES_SCOPEENTRY._serialized_end = 5663 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' + _SWAGGER_RESPONSESENTRY._options = None + _SWAGGER_RESPONSESENTRY._serialized_options = b'8\001' + _SWAGGER_EXTENSIONSENTRY._options = None + _SWAGGER_EXTENSIONSENTRY._serialized_options = b'8\001' + _OPERATION_RESPONSESENTRY._options = None + _OPERATION_RESPONSESENTRY._serialized_options = b'8\001' + _OPERATION_EXTENSIONSENTRY._options = None + _OPERATION_EXTENSIONSENTRY._serialized_options = b'8\001' + _RESPONSE_HEADERSENTRY._options = None + _RESPONSE_HEADERSENTRY._serialized_options = b'8\001' + _RESPONSE_EXAMPLESENTRY._options = None + _RESPONSE_EXAMPLESENTRY._serialized_options = b'8\001' + _RESPONSE_EXTENSIONSENTRY._options = None + _RESPONSE_EXTENSIONSENTRY._serialized_options = b'8\001' + _INFO_EXTENSIONSENTRY._options = None + _INFO_EXTENSIONSENTRY._serialized_options = b'8\001' + _SCHEMA.fields_by_name['example']._options = None + _SCHEMA.fields_by_name['example']._serialized_options = b'\030\001' + _SECURITYDEFINITIONS_SECURITYENTRY._options = None + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_options = b'8\001' + _SECURITYSCHEME_EXTENSIONSENTRY._options = None + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_options = b'8\001' + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._options = None + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_options = b'8\001' + _SCOPES_SCOPEENTRY._options = None + _SCOPES_SCOPEENTRY._serialized_options = b'8\001' + _SWAGGER._serialized_start=145 + _SWAGGER._serialized_end=1073 + _SWAGGER_RESPONSESENTRY._serialized_start=813 + _SWAGGER_RESPONSESENTRY._serialized_end=912 + _SWAGGER_EXTENSIONSENTRY._serialized_start=914 + _SWAGGER_EXTENSIONSENTRY._serialized_end=987 + _SWAGGER_SWAGGERSCHEME._serialized_start=989 + _SWAGGER_SWAGGERSCHEME._serialized_end=1055 + _OPERATION._serialized_start=1076 + _OPERATION._serialized_end=1757 + _OPERATION_RESPONSESENTRY._serialized_start=813 + _OPERATION_RESPONSESENTRY._serialized_end=912 + _OPERATION_EXTENSIONSENTRY._serialized_start=914 + _OPERATION_EXTENSIONSENTRY._serialized_end=987 + _HEADER._serialized_start=1760 + _HEADER._serialized_end=1931 + _RESPONSE._serialized_start=1934 + _RESPONSE._serialized_end=2502 + _RESPONSE_HEADERSENTRY._serialized_start=2283 + _RESPONSE_HEADERSENTRY._serialized_end=2378 + _RESPONSE_EXAMPLESENTRY._serialized_start=2380 + _RESPONSE_EXAMPLESENTRY._serialized_end=2427 + _RESPONSE_EXTENSIONSENTRY._serialized_start=914 + _RESPONSE_EXTENSIONSENTRY._serialized_end=987 + _INFO._serialized_start=2505 + _INFO._serialized_end=2882 + _INFO_EXTENSIONSENTRY._serialized_start=914 + _INFO_EXTENSIONSENTRY._serialized_end=987 + _CONTACT._serialized_start=2884 + _CONTACT._serialized_end=2935 + _LICENSE._serialized_start=2937 + _LICENSE._serialized_end=2973 + _EXTERNALDOCUMENTATION._serialized_start=2975 + _EXTERNALDOCUMENTATION._serialized_end=3032 + _SCHEMA._serialized_start=3035 + _SCHEMA._serialized_end=3319 + _JSONSCHEMA._serialized_start=3322 + _JSONSCHEMA._serialized_end=4061 + _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_start=3864 + _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_end=3983 + _TAG._serialized_start=4063 + _TAG._serialized_end=4182 + _SECURITYDEFINITIONS._serialized_start=4185 + _SECURITYDEFINITIONS._serialized_end=4406 + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_start=4302 + _SECURITYDEFINITIONS_SECURITYENTRY._serialized_end=4406 + _SECURITYSCHEME._serialized_start=4409 + _SECURITYSCHEME._serialized_end=5199 + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_start=914 + _SECURITYSCHEME_EXTENSIONSENTRY._serialized_end=987 + _SECURITYSCHEME_TYPE._serialized_start=4965 + _SECURITYSCHEME_TYPE._serialized_end=5040 + _SECURITYSCHEME_IN._serialized_start=5042 + _SECURITYSCHEME_IN._serialized_end=5091 + _SECURITYSCHEME_FLOW._serialized_start=5093 + _SECURITYSCHEME_FLOW._serialized_end=5199 + _SECURITYREQUIREMENT._serialized_start=5202 + _SECURITYREQUIREMENT._serialized_end=5531 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_start=5342 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_end=5383 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_start=5386 + _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_end=5531 + _SCOPES._serialized_start=5534 + _SCOPES._serialized_end=5663 + _SCOPES_SCOPEENTRY._serialized_start=5619 + _SCOPES_SCOPEENTRY._serialized_end=5663 # @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py index bf947056..2daafffe 100644 --- a/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py +++ b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py @@ -1,4 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc + diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 21ab20a7..c18db761 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.20.6' +__version__ = '1.21.0' diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index 9cb92316..23546c71 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/common/v2/scanoss-common.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -12,25 +11,26 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3' -) + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _STATUSCODE._serialized_start = 336 - _STATUSCODE._serialized_end = 432 - _STATUSRESPONSE._serialized_start = 69 - _STATUSRESPONSE._serialized_end = 153 - _ECHOREQUEST._serialized_start = 155 - _ECHOREQUEST._serialized_end = 185 - _ECHORESPONSE._serialized_start = 187 - _ECHORESPONSE._serialized_end = 218 - _PURLREQUEST._serialized_start = 220 - _PURLREQUEST._serialized_end = 334 - _PURLREQUEST_PURLS._serialized_start = 292 - _PURLREQUEST_PURLS._serialized_end = 334 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' + _STATUSCODE._serialized_start=336 + _STATUSCODE._serialized_end=432 + _STATUSRESPONSE._serialized_start=69 + _STATUSRESPONSE._serialized_end=153 + _ECHOREQUEST._serialized_start=155 + _ECHOREQUEST._serialized_end=185 + _ECHORESPONSE._serialized_start=187 + _ECHORESPONSE._serialized_end=218 + _PURLREQUEST._serialized_start=220 + _PURLREQUEST._serialized_end=334 + _PURLREQUEST_PURLS._serialized_start=292 + _PURLREQUEST_PURLS._serialized_end=334 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index bf947056..2daafffe 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -1,4 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc + diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index 04938b6d..cf1290a9 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/components/v2/scanoss-components.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -17,55 +16,46 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xd4\x04\n\nComponents\x12s\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse""\x82\xd3\xe4\x93\x02\x1c"\x17/api/v2/components/echo:\x01*\x12\x95\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse"$\x82\xd3\xe4\x93\x02\x1e"\x19/api/v2/components/search:\x01*\x12\x9d\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse"&\x82\xd3\xe4\x93\x02 "\x1b/api/v2/components/versions:\x01*\x12\x98\x01\n\x16GetComponentStatistics\x12".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse"(\x82\xd3\xe4\x93\x02""\x1d/api/v2/components/statistics:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Components Service"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05\"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xd4\x04\n\nComponents\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/components/echo:\x01*\x12\x95\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/components/search:\x01*\x12\x9d\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"&\x82\xd3\xe4\x93\x02 \"\x1b/api/v2/components/versions:\x01*\x12\x98\x01\n\x16GetComponentStatistics\x12\".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse\"(\x82\xd3\xe4\x93\x02\"\"\x1d/api/v2/components/statistics:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Components Service\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\331\001\022s\n\032SCANOSS Components Service"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _COMPONENTS.methods_by_name['Echo']._options = None - _COMPONENTS.methods_by_name[ - 'Echo' - ]._serialized_options = b'\202\323\344\223\002\034"\027/api/v2/components/echo:\001*' - _COMPONENTS.methods_by_name['SearchComponents']._options = None - _COMPONENTS.methods_by_name[ - 'SearchComponents' - ]._serialized_options = b'\202\323\344\223\002\036"\031/api/v2/components/search:\001*' - _COMPONENTS.methods_by_name['GetComponentVersions']._options = None - _COMPONENTS.methods_by_name[ - 'GetComponentVersions' - ]._serialized_options = b'\202\323\344\223\002 "\033/api/v2/components/versions:\001*' - _COMPONENTS.methods_by_name['GetComponentStatistics']._options = None - _COMPONENTS.methods_by_name[ - 'GetComponentStatistics' - ]._serialized_options = b'\202\323\344\223\002""\035/api/v2/components/statistics:\001*' - _COMPSEARCHREQUEST._serialized_start = 201 - _COMPSEARCHREQUEST._serialized_end = 319 - _COMPSTATISTIC._serialized_start = 322 - _COMPSTATISTIC._serialized_end = 524 - _COMPSTATISTIC_LANGUAGE._serialized_start = 485 - _COMPSTATISTIC_LANGUAGE._serialized_end = 524 - _COMPSTATISTICRESPONSE._serialized_start = 527 - _COMPSTATISTICRESPONSE._serialized_end = 778 - _COMPSTATISTICRESPONSE_PURLS._serialized_start = 678 - _COMPSTATISTICRESPONSE_PURLS._serialized_end = 778 - _COMPSEARCHRESPONSE._serialized_start = 781 - _COMPSEARCHRESPONSE._serialized_end = 992 - _COMPSEARCHRESPONSE_COMPONENT._serialized_start = 935 - _COMPSEARCHRESPONSE_COMPONENT._serialized_end = 992 - _COMPVERSIONREQUEST._serialized_start = 994 - _COMPVERSIONREQUEST._serialized_end = 1043 - _COMPVERSIONRESPONSE._serialized_start = 1046 - _COMPVERSIONRESPONSE._serialized_end = 1516 - _COMPVERSIONRESPONSE_LICENSE._serialized_start = 1201 - _COMPVERSIONRESPONSE_LICENSE._serialized_end = 1280 - _COMPVERSIONRESPONSE_VERSION._serialized_start = 1282 - _COMPVERSIONRESPONSE_VERSION._serialized_end = 1382 - _COMPVERSIONRESPONSE_COMPONENT._serialized_start = 1385 - _COMPVERSIONRESPONSE_COMPONENT._serialized_end = 1516 - _COMPONENTS._serialized_start = 1519 - _COMPONENTS._serialized_end = 2115 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\331\001\022s\n\032SCANOSS Components Service\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _COMPONENTS.methods_by_name['Echo']._options = None + _COMPONENTS.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/components/echo:\001*' + _COMPONENTS.methods_by_name['SearchComponents']._options = None + _COMPONENTS.methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/components/search:\001*' + _COMPONENTS.methods_by_name['GetComponentVersions']._options = None + _COMPONENTS.methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002 \"\033/api/v2/components/versions:\001*' + _COMPONENTS.methods_by_name['GetComponentStatistics']._options = None + _COMPONENTS.methods_by_name['GetComponentStatistics']._serialized_options = b'\202\323\344\223\002\"\"\035/api/v2/components/statistics:\001*' + _COMPSEARCHREQUEST._serialized_start=201 + _COMPSEARCHREQUEST._serialized_end=319 + _COMPSTATISTIC._serialized_start=322 + _COMPSTATISTIC._serialized_end=524 + _COMPSTATISTIC_LANGUAGE._serialized_start=485 + _COMPSTATISTIC_LANGUAGE._serialized_end=524 + _COMPSTATISTICRESPONSE._serialized_start=527 + _COMPSTATISTICRESPONSE._serialized_end=778 + _COMPSTATISTICRESPONSE_PURLS._serialized_start=678 + _COMPSTATISTICRESPONSE_PURLS._serialized_end=778 + _COMPSEARCHRESPONSE._serialized_start=781 + _COMPSEARCHRESPONSE._serialized_end=992 + _COMPSEARCHRESPONSE_COMPONENT._serialized_start=935 + _COMPSEARCHRESPONSE_COMPONENT._serialized_end=992 + _COMPVERSIONREQUEST._serialized_start=994 + _COMPVERSIONREQUEST._serialized_end=1043 + _COMPVERSIONRESPONSE._serialized_start=1046 + _COMPVERSIONRESPONSE._serialized_end=1516 + _COMPVERSIONRESPONSE_LICENSE._serialized_start=1201 + _COMPVERSIONRESPONSE_LICENSE._serialized_end=1280 + _COMPVERSIONRESPONSE_VERSION._serialized_start=1282 + _COMPVERSIONRESPONSE_VERSION._serialized_end=1382 + _COMPVERSIONRESPONSE_COMPONENT._serialized_start=1385 + _COMPVERSIONRESPONSE_COMPONENT._serialized_end=1516 + _COMPONENTS._serialized_start=1519 + _COMPONENTS._serialized_end=2115 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py index da8d455a..8ac7b92c 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py @@ -1,12 +1,9 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from scanoss.api.components.v2 import ( - scanoss_components_pb2 as scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2, -) +from scanoss.api.components.v2 import scanoss_components_pb2 as scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2 class ComponentsStub(object): @@ -21,25 +18,25 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.components.v2.Components/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.components.v2.Components/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.SearchComponents = channel.unary_unary( - '/scanoss.api.components.v2.Components/SearchComponents', - request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, - ) + '/scanoss.api.components.v2.Components/SearchComponents', + request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, + ) self.GetComponentVersions = channel.unary_unary( - '/scanoss.api.components.v2.Components/GetComponentVersions', - request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, - ) + '/scanoss.api.components.v2.Components/GetComponentVersions', + request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, + ) self.GetComponentStatistics = channel.unary_unary( - '/scanoss.api.components.v2.Components/GetComponentStatistics', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, - ) + '/scanoss.api.components.v2.Components/GetComponentStatistics', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, + ) class ComponentsServicer(object): @@ -48,25 +45,29 @@ class ComponentsServicer(object): """ def Echo(self, request, context): - """Standard echo""" + """Standard echo + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def SearchComponents(self, request, context): - """Search for components""" + """Search for components + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetComponentVersions(self, request, context): - """Get all version information for a specific component""" + """Get all version information for a specific component + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetComponentStatistics(self, request, context): - """Get the statistics for the specified components""" + """Get the statistics for the specified components + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -74,149 +75,102 @@ def GetComponentStatistics(self, request, context): def add_ComponentsServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'SearchComponents': grpc.unary_unary_rpc_method_handler( - servicer.SearchComponents, - request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.FromString, - response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.SerializeToString, - ), - 'GetComponentVersions': grpc.unary_unary_rpc_method_handler( - servicer.GetComponentVersions, - request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.FromString, - response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.SerializeToString, - ), - 'GetComponentStatistics': grpc.unary_unary_rpc_method_handler( - servicer.GetComponentStatistics, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'SearchComponents': grpc.unary_unary_rpc_method_handler( + servicer.SearchComponents, + request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.SerializeToString, + ), + 'GetComponentVersions': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentVersions, + request_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.SerializeToString, + ), + 'GetComponentStatistics': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentStatistics, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.SerializeToString, + ), } - generic_handler = grpc.method_handlers_generic_handler('scanoss.api.components.v2.Components', rpc_method_handlers) + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.components.v2.Components', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) -# This class is part of an EXPERIMENTAL API. + # This class is part of an EXPERIMENTAL API. class Components(object): """ Expose all of the SCANOSS Component RPCs here """ @staticmethod - def Echo( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def Echo(request, target, - '/scanoss.api.components.v2.Components/Echo', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def SearchComponents( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def SearchComponents(request, target, - '/scanoss.api.components.v2.Components/SearchComponents', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/SearchComponents', scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetComponentVersions( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def GetComponentVersions(request, target, - '/scanoss.api.components.v2.Components/GetComponentVersions', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/GetComponentVersions', scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetComponentStatistics( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def GetComponentStatistics(request, target, - '/scanoss.api.components.v2.Components/GetComponentStatistics', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/GetComponentStatistics', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py index 577b9af2..f5af2f39 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/cryptography/v2/scanoss-cryptography.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -17,29 +16,50 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"\xb9\x02\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x31\n\nAlgorithms\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\x1au\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12M\n\nalgorithms\x18\x03 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.AlgorithmResponse.Algorithms2\x97\x02\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse"$\x82\xd3\xe4\x93\x02\x1e"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse"*\x82\xd3\xe4\x93\x02$"\x1f/api/v2/cryptography/algorithms:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"0\n\tAlgorithm\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\"\xf3\x01\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\x82\x02\n\x19\x41lgorithmsInRangeResponse\x12J\n\x05purls\x18\x01 \x03(\x0b\x32;.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\xe1\x01\n\x17VersionsInRangeResponse\x12H\n\x05purls\x18\x01 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.VersionsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x45\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x15\n\rversions_with\x18\x02 \x03(\t\x12\x18\n\x10versions_without\x18\x03 \x03(\t\"}\n\x04Hint\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08\x63\x61tegory\x18\x04 \x01(\t\x12\x10\n\x03url\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x11\n\x04purl\x18\x06 \x01(\tH\x01\x88\x01\x01\x42\x06\n\x04_urlB\x07\n\x05_purl\"\xe1\x01\n\rHintsResponse\x12?\n\x05purls\x18\x01 \x03(\x0b\x32\x30.scanoss.api.cryptography.v2.HintsResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\xee\x01\n\x14HintsInRangeResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.cryptography.v2.HintsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint2\x88\x07\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/cryptography/algorithms:\x01*\x12\xa5\x01\n\x14GetAlgorithmsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x36.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse\"1\x82\xd3\xe4\x93\x02+\"&/api/v2/cryptography/algorithmsInRange:\x01*\x12\x9f\x01\n\x12GetVersionsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x34.scanoss.api.cryptography.v2.VersionsInRangeResponse\"/\x82\xd3\xe4\x93\x02)\"$/api/v2/cryptography/versionsInRange:\x01*\x12\x96\x01\n\x0fGetHintsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.cryptography.v2.HintsInRangeResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/cryptography/hintsInRange:\x01*\x12\x8b\x01\n\x12GetEncryptionHints\x12\".scanoss.api.common.v2.PurlRequest\x1a*.scanoss.api.cryptography.v2.HintsResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/cryptography/hints:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _CRYPTOGRAPHY.methods_by_name['Echo']._options = None - _CRYPTOGRAPHY.methods_by_name[ - 'Echo' - ]._serialized_options = b'\202\323\344\223\002\036"\031/api/v2/cryptography/echo:\001*' - _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._options = None - _CRYPTOGRAPHY.methods_by_name[ - 'GetAlgorithms' - ]._serialized_options = b'\202\323\344\223\002$"\037/api/v2/cryptography/algorithms:\001*' - _ALGORITHMRESPONSE._serialized_start = 208 - _ALGORITHMRESPONSE._serialized_end = 521 - _ALGORITHMRESPONSE_ALGORITHMS._serialized_start = 353 - _ALGORITHMRESPONSE_ALGORITHMS._serialized_end = 402 - _ALGORITHMRESPONSE_PURLS._serialized_start = 404 - _ALGORITHMRESPONSE_PURLS._serialized_end = 521 - _CRYPTOGRAPHY._serialized_start = 524 - _CRYPTOGRAPHY._serialized_end = 803 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _CRYPTOGRAPHY.methods_by_name['Echo']._options = None + _CRYPTOGRAPHY.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/cryptography/echo:\001*' + _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._options = None + _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/cryptography/algorithms:\001*' + _CRYPTOGRAPHY.methods_by_name['GetAlgorithmsInRange']._options = None + _CRYPTOGRAPHY.methods_by_name['GetAlgorithmsInRange']._serialized_options = b'\202\323\344\223\002+\"&/api/v2/cryptography/algorithmsInRange:\001*' + _CRYPTOGRAPHY.methods_by_name['GetVersionsInRange']._options = None + _CRYPTOGRAPHY.methods_by_name['GetVersionsInRange']._serialized_options = b'\202\323\344\223\002)\"$/api/v2/cryptography/versionsInRange:\001*' + _CRYPTOGRAPHY.methods_by_name['GetHintsInRange']._options = None + _CRYPTOGRAPHY.methods_by_name['GetHintsInRange']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/cryptography/hintsInRange:\001*' + _CRYPTOGRAPHY.methods_by_name['GetEncryptionHints']._options = None + _CRYPTOGRAPHY.methods_by_name['GetEncryptionHints']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/cryptography/hints:\001*' + _ALGORITHM._serialized_start=207 + _ALGORITHM._serialized_end=255 + _ALGORITHMRESPONSE._serialized_start=258 + _ALGORITHMRESPONSE._serialized_end=501 + _ALGORITHMRESPONSE_PURLS._serialized_start=403 + _ALGORITHMRESPONSE_PURLS._serialized_end=501 + _ALGORITHMSINRANGERESPONSE._serialized_start=504 + _ALGORITHMSINRANGERESPONSE._serialized_end=762 + _ALGORITHMSINRANGERESPONSE_PURL._serialized_start=664 + _ALGORITHMSINRANGERESPONSE_PURL._serialized_end=762 + _VERSIONSINRANGERESPONSE._serialized_start=765 + _VERSIONSINRANGERESPONSE._serialized_end=990 + _VERSIONSINRANGERESPONSE_PURL._serialized_start=921 + _VERSIONSINRANGERESPONSE_PURL._serialized_end=990 + _HINT._serialized_start=992 + _HINT._serialized_end=1117 + _HINTSRESPONSE._serialized_start=1120 + _HINTSRESPONSE._serialized_end=1345 + _HINTSRESPONSE_PURLS._serialized_start=1257 + _HINTSRESPONSE_PURLS._serialized_end=1345 + _HINTSINRANGERESPONSE._serialized_start=1348 + _HINTSINRANGERESPONSE._serialized_end=1586 + _HINTSINRANGERESPONSE_PURL._serialized_start=1498 + _HINTSINRANGERESPONSE_PURL._serialized_end=1586 + _CRYPTOGRAPHY._serialized_start=1589 + _CRYPTOGRAPHY._serialized_end=2493 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py index 22a866a8..25b77e2c 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py @@ -1,12 +1,9 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from scanoss.api.cryptography.v2 import ( - scanoss_cryptography_pb2 as scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2, -) +from scanoss.api.cryptography.v2 import scanoss_cryptography_pb2 as scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2 class CryptographyStub(object): @@ -21,15 +18,35 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.cryptography.v2.Cryptography/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.cryptography.v2.Cryptography/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.GetAlgorithms = channel.unary_unary( - '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, - ) + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, + ) + self.GetAlgorithmsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithmsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmsInRangeResponse.FromString, + ) + self.GetVersionsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetVersionsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.VersionsInRangeResponse.FromString, + ) + self.GetHintsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetHintsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsInRangeResponse.FromString, + ) + self.GetEncryptionHints = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetEncryptionHints', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsResponse.FromString, + ) class CryptographyServicer(object): @@ -38,13 +55,43 @@ class CryptographyServicer(object): """ def Echo(self, request, context): - """Standard echo""" + """Standard echo + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetAlgorithms(self, request, context): - """Get Cryptographic algorithms associated with a list of PURLs""" + """Get Cryptographic algorithms associated with a list of PURLs and, optionally, a requirement + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetAlgorithmsInRange(self, request, context): + """Given a list of PURLS and version ranges, get a list of cryptographic algorithms used + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetVersionsInRange(self, request, context): + """Given a list of PURLS and version ranges, get a list of versions that do/do not contain cryptographic algorithms + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetHintsInRange(self, request, context): + """Given a list of PURLS and version ranges, get hints related to protocol/library/sdk/framework + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetEncryptionHints(self, request, context): + """Given a list of PURLS, get hints related to protocol/library/sdk/framework + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -52,83 +99,146 @@ def GetAlgorithms(self, request, context): def add_CryptographyServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'GetAlgorithms': grpc.unary_unary_rpc_method_handler( - servicer.GetAlgorithms, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetAlgorithms': grpc.unary_unary_rpc_method_handler( + servicer.GetAlgorithms, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.SerializeToString, + ), + 'GetAlgorithmsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetAlgorithmsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmsInRangeResponse.SerializeToString, + ), + 'GetVersionsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetVersionsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.VersionsInRangeResponse.SerializeToString, + ), + 'GetHintsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetHintsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsInRangeResponse.SerializeToString, + ), + 'GetEncryptionHints': grpc.unary_unary_rpc_method_handler( + servicer.GetEncryptionHints, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers - ) + 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) -# This class is part of an EXPERIMENTAL API. + # This class is part of an EXPERIMENTAL API. class Cryptography(object): """ Expose all of the SCANOSS Cryptography RPCs here """ @staticmethod - def Echo( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def Echo(request, target, - '/scanoss.api.cryptography.v2.Cryptography/Echo', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetAlgorithms( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def GetAlgorithms(request, target, - '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetAlgorithmsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithmsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmsInRangeResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetVersionsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetVersionsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.VersionsInRangeResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetHintsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetHintsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsInRangeResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetEncryptionHints(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetEncryptionHints', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index c1915e39..0090b4cd 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/dependencies/v2/scanoss-dependencies.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -17,37 +16,32 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xa8\x02\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse"$\x82\xd3\xe4\x93\x02\x1e"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse",\x82\xd3\xe4\x93\x02&"!/api/v2/dependencies/dependencies:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xa8\x02\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _DEPENDENCIES.methods_by_name['Echo']._options = None - _DEPENDENCIES.methods_by_name[ - 'Echo' - ]._serialized_options = b'\202\323\344\223\002\036"\031/api/v2/dependencies/echo:\001*' - _DEPENDENCIES.methods_by_name['GetDependencies']._options = None - _DEPENDENCIES.methods_by_name[ - 'GetDependencies' - ]._serialized_options = b'\202\323\344\223\002&"!/api/v2/dependencies/dependencies:\001*' - _DEPENDENCYREQUEST._serialized_start = 208 - _DEPENDENCYREQUEST._serialized_end = 447 - _DEPENDENCYREQUEST_PURLS._serialized_start = 313 - _DEPENDENCYREQUEST_PURLS._serialized_end = 355 - _DEPENDENCYREQUEST_FILES._serialized_start = 357 - _DEPENDENCYREQUEST_FILES._serialized_end = 447 - _DEPENDENCYRESPONSE._serialized_start = 450 - _DEPENDENCYRESPONSE._serialized_end = 986 - _DEPENDENCYRESPONSE_LICENSES._serialized_start = 597 - _DEPENDENCYRESPONSE_LICENSES._serialized_end = 677 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start = 680 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end = 850 - _DEPENDENCYRESPONSE_FILES._serialized_start = 853 - _DEPENDENCYRESPONSE_FILES._serialized_end = 986 - _DEPENDENCIES._serialized_start = 989 - _DEPENDENCIES._serialized_end = 1285 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service\"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _DEPENDENCIES.methods_by_name['Echo']._options = None + _DEPENDENCIES.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' + _DEPENDENCIES.methods_by_name['GetDependencies']._options = None + _DEPENDENCIES.methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' + _DEPENDENCYREQUEST._serialized_start=208 + _DEPENDENCYREQUEST._serialized_end=447 + _DEPENDENCYREQUEST_PURLS._serialized_start=313 + _DEPENDENCYREQUEST_PURLS._serialized_end=355 + _DEPENDENCYREQUEST_FILES._serialized_start=357 + _DEPENDENCYREQUEST_FILES._serialized_end=447 + _DEPENDENCYRESPONSE._serialized_start=450 + _DEPENDENCYRESPONSE._serialized_end=986 + _DEPENDENCYRESPONSE_LICENSES._serialized_start=597 + _DEPENDENCYRESPONSE_LICENSES._serialized_end=677 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=680 + _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=850 + _DEPENDENCYRESPONSE_FILES._serialized_start=853 + _DEPENDENCYRESPONSE_FILES._serialized_end=986 + _DEPENDENCIES._serialized_start=989 + _DEPENDENCIES._serialized_end=1285 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py index 53d610f7..cddf7cfa 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py @@ -1,12 +1,9 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from scanoss.api.dependencies.v2 import ( - scanoss_dependencies_pb2 as scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2, -) +from scanoss.api.dependencies.v2 import scanoss_dependencies_pb2 as scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2 class DependenciesStub(object): @@ -21,15 +18,15 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.dependencies.v2.Dependencies/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.dependencies.v2.Dependencies/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.GetDependencies = channel.unary_unary( - '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', - request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, - ) + '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', + request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, + ) class DependenciesServicer(object): @@ -38,13 +35,15 @@ class DependenciesServicer(object): """ def Echo(self, request, context): - """Standard echo""" + """Standard echo + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetDependencies(self, request, context): - """Get dependency details""" + """Get dependency details + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -52,83 +51,58 @@ def GetDependencies(self, request, context): def add_DependenciesServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'GetDependencies': grpc.unary_unary_rpc_method_handler( - servicer.GetDependencies, - request_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.FromString, - response_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetDependencies': grpc.unary_unary_rpc_method_handler( + servicer.GetDependencies, + request_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.FromString, + response_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.dependencies.v2.Dependencies', rpc_method_handlers - ) + 'scanoss.api.dependencies.v2.Dependencies', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) -# This class is part of an EXPERIMENTAL API. + # This class is part of an EXPERIMENTAL API. class Dependencies(object): """ Expose all of the SCANOSS Dependency RPCs here """ @staticmethod - def Echo( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def Echo(request, target, - '/scanoss.api.dependencies.v2.Dependencies/Echo', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetDependencies( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def GetDependencies(request, target, - '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py index 3bbb92b8..5af1b6b2 100644 --- a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py +++ b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/provenance/v2/scanoss-provenance.proto -# Protobuf Python Version: 4.25.1 """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -19,24 +18,24 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/provenance/v2/scanoss-provenance.proto\x12\x19scanoss.api.provenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xd5\x03\n\x12ProvenanceResponse\x12\x42\n\x05purls\x18\x01 \x03(\x0b\x32\x33.scanoss.api.provenance.v2.ProvenanceResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xdc\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12Z\n\x12\x64\x65\x63lared_locations\x18\x03 \x03(\x0b\x32>.scanoss.api.provenance.v2.ProvenanceResponse.DeclaredLocation\x12X\n\x11\x63urated_locations\x18\x04 \x03(\x0b\x32=.scanoss.api.provenance.v2.ProvenanceResponse.CuratedLocation2\x98\x02\n\nProvenance\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/provenance/echo:\x01*\x12\x94\x01\n\x16GetComponentProvenance\x12\".scanoss.api.common.v2.PurlRequest\x1a-.scanoss.api.provenance.v2.ProvenanceResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/provenance/countries:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/provenancev2;provenancev2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Provenance Service\"P\n\x12scanoss-provenance\x12%https://github.com/scanoss/provenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.provenance.v2.scanoss_provenance_pb2', _globals) +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.provenance.v2.scanoss_provenance_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - _globals['DESCRIPTOR']._options = None - _globals['DESCRIPTOR']._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/provenancev2;provenancev2\222A\331\001\022s\n\032SCANOSS Provenance Service\"P\n\022scanoss-provenance\022%https://github.com/scanoss/provenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _globals['_PROVENANCE'].methods_by_name['Echo']._options = None - _globals['_PROVENANCE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/provenance/echo:\001*' - _globals['_PROVENANCE'].methods_by_name['GetComponentProvenance']._options = None - _globals['_PROVENANCE'].methods_by_name['GetComponentProvenance']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/provenance/countries:\001*' - _globals['_PROVENANCERESPONSE']._serialized_start=202 - _globals['_PROVENANCERESPONSE']._serialized_end=671 - _globals['_PROVENANCERESPONSE_DECLAREDLOCATION']._serialized_start=347 - _globals['_PROVENANCERESPONSE_DECLAREDLOCATION']._serialized_end=397 - _globals['_PROVENANCERESPONSE_CURATEDLOCATION']._serialized_start=399 - _globals['_PROVENANCERESPONSE_CURATEDLOCATION']._serialized_end=448 - _globals['_PROVENANCERESPONSE_PURLS']._serialized_start=451 - _globals['_PROVENANCERESPONSE_PURLS']._serialized_end=671 - _globals['_PROVENANCE']._serialized_start=674 - _globals['_PROVENANCE']._serialized_end=954 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/provenancev2;provenancev2\222A\331\001\022s\n\032SCANOSS Provenance Service\"P\n\022scanoss-provenance\022%https://github.com/scanoss/provenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _PROVENANCE.methods_by_name['Echo']._options = None + _PROVENANCE.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/provenance/echo:\001*' + _PROVENANCE.methods_by_name['GetComponentProvenance']._options = None + _PROVENANCE.methods_by_name['GetComponentProvenance']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/provenance/countries:\001*' + _PROVENANCERESPONSE._serialized_start=202 + _PROVENANCERESPONSE._serialized_end=671 + _PROVENANCERESPONSE_DECLAREDLOCATION._serialized_start=347 + _PROVENANCERESPONSE_DECLAREDLOCATION._serialized_end=397 + _PROVENANCERESPONSE_CURATEDLOCATION._serialized_start=399 + _PROVENANCERESPONSE_CURATEDLOCATION._serialized_end=448 + _PROVENANCERESPONSE_PURLS._serialized_start=451 + _PROVENANCERESPONSE_PURLS._serialized_end=671 + _PROVENANCE._serialized_start=674 + _PROVENANCE._serialized_end=954 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py index 7143ca60..1be8745d 100644 --- a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py +++ b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py @@ -42,7 +42,7 @@ def Echo(self, request, context): raise NotImplementedError('Method not implemented!') def GetComponentProvenance(self, request, context): - """Get Provenance countrues associated with a list of PURLs + """Get Provenance countries associated with a list of PURLs """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 48331ce3..af611a33 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/scanning/v2/scanoss-scanning.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -17,17 +16,28 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto2}\n\x08Scanning\x12q\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse" \x82\xd3\xe4\x93\x02\x1a"\x15/api/v2/scanning/echo:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xff\x01\n\nHFHRequest\x12\x12\n\nbest_match\x18\x01 \x01(\x08\x12\x11\n\tthreshold\x18\x02 \x01(\x05\x12:\n\x04root\x18\x03 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x1a\x8d\x01\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\"\xa2\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a?\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x12\n\nconfidence\x18\x03 \x01(\x02\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _SCANNING.methods_by_name['Echo']._options = None - _SCANNING.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032"\025/api/v2/scanning/echo:\001*' - _SCANNING._serialized_start = 195 - _SCANNING._serialized_end = 320 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service\"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _SCANNING.methods_by_name['Echo']._options = None + _SCANNING.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' + _SCANNING.methods_by_name['FolderHashScan']._options = None + _SCANNING.methods_by_name['FolderHashScan']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/scanning/hfh/scan:\001*' + _HFHREQUEST._serialized_start=196 + _HFHREQUEST._serialized_end=451 + _HFHREQUEST_CHILDREN._serialized_start=310 + _HFHREQUEST_CHILDREN._serialized_end=451 + _HFHRESPONSE._serialized_start=454 + _HFHRESPONSE._serialized_end=744 + _HFHRESPONSE_COMPONENT._serialized_start=586 + _HFHRESPONSE_COMPONENT._serialized_end=649 + _HFHRESPONSE_RESULT._serialized_start=651 + _HFHRESPONSE_RESULT._serialized_end=744 + _SCANNING._serialized_start=747 + _SCANNING._serialized_end=1004 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index 9bf113b2..d00b7e38 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -1,13 +1,15 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from scanoss.api.scanning.v2 import scanoss_scanning_pb2 as scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2 class ScanningStub(object): - """Expose all of the SCANOSS Scanning RPCs here""" + """* + Expose all of the SCANOSS Scanning RPCs here + """ def __init__(self, channel): """Constructor. @@ -16,17 +18,32 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.scanning.v2.Scanning/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.scanning.v2.Scanning/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) + self.FolderHashScan = channel.unary_unary( + '/scanoss.api.scanning.v2.Scanning/FolderHashScan', + request_serializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, + ) class ScanningServicer(object): - """Expose all of the SCANOSS Scanning RPCs here""" + """* + Expose all of the SCANOSS Scanning RPCs here + """ def Echo(self, request, context): - """Standard echo""" + """Standard echo + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def FolderHashScan(self, request, context): + """Scan the given folder request looking for matches + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -34,45 +51,58 @@ def Echo(self, request, context): def add_ScanningServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'FolderHashScan': grpc.unary_unary_rpc_method_handler( + servicer.FolderHashScan, + request_deserializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.FromString, + response_serializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.SerializeToString, + ), } - generic_handler = grpc.method_handlers_generic_handler('scanoss.api.scanning.v2.Scanning', rpc_method_handlers) + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.scanning.v2.Scanning', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) -# This class is part of an EXPERIMENTAL API. + # This class is part of an EXPERIMENTAL API. class Scanning(object): - """Expose all of the SCANOSS Scanning RPCs here""" + """* + Expose all of the SCANOSS Scanning RPCs here + """ @staticmethod - def Echo( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def Echo(request, target, - '/scanoss.api.scanning.v2.Scanning/Echo', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def FolderHashScan(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/FolderHashScan', + scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, + scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py index a3550b90..1b8b0461 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/semgrep/v2/scanoss-semgrep.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -17,29 +16,26 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse"\x1f\x82\xd3\xe4\x93\x02\x19"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse"!\x82\xd3\xe4\x93\x02\x1b"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service"J\n\x0fscanoss-semgrep\x12"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3' -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12\".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse\"!\x82\xd3\xe4\x93\x02\x1b\"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service\"J\n\x0fscanoss-semgrep\x12\"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.semgrep.v2.scanoss_semgrep_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service"J\n\017scanoss-semgrep\022"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _SEMGREP.methods_by_name['Echo']._options = None - _SEMGREP.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031"\024/api/v2/semgrep/echo:\001*' - _SEMGREP.methods_by_name['GetIssues']._options = None - _SEMGREP.methods_by_name[ - 'GetIssues' - ]._serialized_options = b'\202\323\344\223\002\033"\026/api/v2/semgrep/issues:\001*' - _SEMGREPRESPONSE._serialized_start = 193 - _SEMGREPRESPONSE._serialized_end = 599 - _SEMGREPRESPONSE_ISSUE._serialized_start = 329 - _SEMGREPRESPONSE_ISSUE._serialized_end = 396 - _SEMGREPRESPONSE_FILE._serialized_start = 398 - _SEMGREPRESPONSE_FILE._serialized_end = 498 - _SEMGREPRESPONSE_PURLS._serialized_start = 500 - _SEMGREPRESPONSE_PURLS._serialized_end = 599 - _SEMGREP._serialized_start = 602 - _SEMGREP._serialized_end = 850 + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service\"J\n\017scanoss-semgrep\022\"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _SEMGREP.methods_by_name['Echo']._options = None + _SEMGREP.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\"\024/api/v2/semgrep/echo:\001*' + _SEMGREP.methods_by_name['GetIssues']._options = None + _SEMGREP.methods_by_name['GetIssues']._serialized_options = b'\202\323\344\223\002\033\"\026/api/v2/semgrep/issues:\001*' + _SEMGREPRESPONSE._serialized_start=193 + _SEMGREPRESPONSE._serialized_end=599 + _SEMGREPRESPONSE_ISSUE._serialized_start=329 + _SEMGREPRESPONSE_ISSUE._serialized_end=396 + _SEMGREPRESPONSE_FILE._serialized_start=398 + _SEMGREPRESPONSE_FILE._serialized_end=498 + _SEMGREPRESPONSE_PURLS._serialized_start=500 + _SEMGREPRESPONSE_PURLS._serialized_end=599 + _SEMGREP._serialized_start=602 + _SEMGREP._serialized_end=850 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py index d11ba8f0..4748a3ee 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py @@ -1,6 +1,5 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" - import grpc from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 @@ -19,15 +18,15 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.semgrep.v2.Semgrep/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + '/scanoss.api.semgrep.v2.Semgrep/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) self.GetIssues = channel.unary_unary( - '/scanoss.api.semgrep.v2.Semgrep/GetIssues', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, - ) + '/scanoss.api.semgrep.v2.Semgrep/GetIssues', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, + ) class SemgrepServicer(object): @@ -36,13 +35,15 @@ class SemgrepServicer(object): """ def Echo(self, request, context): - """Standard echo""" + """Standard echo + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetIssues(self, request, context): - """Get Potential issues associated with a list of PURLs""" + """Get Potential issues associated with a list of PURLs + """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') @@ -50,81 +51,58 @@ def GetIssues(self, request, context): def add_SemgrepServicer_to_server(servicer, server): rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'GetIssues': grpc.unary_unary_rpc_method_handler( - servicer.GetIssues, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.SerializeToString, - ), + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetIssues': grpc.unary_unary_rpc_method_handler( + servicer.GetIssues, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.SerializeToString, + ), } - generic_handler = grpc.method_handlers_generic_handler('scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) -# This class is part of an EXPERIMENTAL API. + # This class is part of an EXPERIMENTAL API. class Semgrep(object): """ Expose all of the SCANOSS Cryptography RPCs here """ @staticmethod - def Echo( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def Echo(request, target, - '/scanoss.api.semgrep.v2.Semgrep/Echo', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def GetIssues( - request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None, - ): - return grpc.experimental.unary_unary( - request, + def GetIssues(request, target, - '/scanoss.api.semgrep.v2.Semgrep/GetIssues', + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/GetIssues', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - ) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index 9b4ea185..9fc87ed3 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -17,43 +16,34 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n None: # noqa: PLR0915 help='Timeout (in seconds) for API communication (optional - default 180)', ) p_scan.add_argument( - '--retry', '-R', type=int, default=DEFAULT_RETRY, - help='Retry limit for API communication (optional - default 5)' + '--retry', + '-R', + type=int, + default=DEFAULT_RETRY, + help='Retry limit for API communication (optional - default 5)', ) p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') p_scan.add_argument( - '--sc-command', type=str, - help='Scancode command and path if required (optional - default scancode).' + '--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' ) p_scan.add_argument( '--sc-timeout', @@ -153,18 +176,6 @@ def setup_args() -> None: # noqa: PLR0915 ) p_scan.add_argument('--dep-scope-inc', '-dsi', type=str, help='Include dependencies with declared scopes') p_scan.add_argument('--dep-scope-exc', '-dse', type=str, help='Exclude dependencies with declared scopes') - p_scan.add_argument( - '--settings', - '-st', - type=str, - help='Settings file to use for scanning (optional - default scanoss.json)', - ) - p_scan.add_argument( - '--skip-settings-file', - '-stf', - action='store_true', - help='Skip default settings file (scanoss.json) if it exists', - ) # Sub-command: fingerprint p_wfp = subparsers.add_parser( @@ -183,18 +194,6 @@ def setup_args() -> None: # noqa: PLR0915 help='Fingerprint the file contents supplied via STDIN (optional)', ) p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') - p_wfp.add_argument( - '--settings', - '-st', - type=str, - help='Settings file to use for fingerprinting (optional - default scanoss.json)', - ) - p_wfp.add_argument( - '--skip-settings-file', - '-stf', - action='store_true', - help='Skip default settings file (scanoss.json) if it exists', - ) # Sub-command: dependency p_dep = subparsers.add_parser( @@ -333,6 +332,7 @@ def setup_args() -> None: # noqa: PLR0915 for p in [c_crypto, c_vulns, c_semgrep, c_provenance]: p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') + # Common Component sub-command options for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -491,6 +491,81 @@ def setup_args() -> None: # noqa: PLR0915 ) p_undeclared.set_defaults(func=inspect_undeclared) + # Sub-command: folder-scan + p_folder_scan = subparsers.add_parser( + 'folder-scan', + aliases=['fs'], + description=f'Scan the given directory using folder hashing: {__version__}', + help='Scan the given directory using folder hashing', + ) + p_folder_scan.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='The root directory to scan') + p_folder_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + p_folder_scan.add_argument( + '--timeout', + '-M', + type=int, + default=600, + help='Timeout (in seconds) for API communication (optional - default 600)', + ) + p_folder_scan.add_argument( + '--format', + '-f', + type=str, + choices=['json'], + default='json', + help='Result output format (optional - default: json)', + ) + p_folder_scan.add_argument( + '--best-match', + '-bm', + action='store_true', + default=False, + help='Enable best match mode (optional - default: False)', + ) + p_folder_scan.add_argument( + '--threshold', + type=int, + choices=range(1, 101), + metavar='1-100', + default=100, + help='Threshold for result matching (optional - default: 100)', + ) + p_folder_scan.set_defaults(func=folder_hashing_scan) + + # Sub-command: folder-hash + p_folder_hash = subparsers.add_parser( + 'folder-hash', + aliases=['fh'], + description=f'Produce a folder hash for the given directory: {__version__}', + help='Produce a folder hash for the given directory', + ) + p_folder_hash.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') + p_folder_hash.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + p_folder_hash.add_argument( + '--format', + '-f', + type=str, + choices=['json'], + default='json', + help='Result output format (optional - default: json)', + ) + p_folder_hash.set_defaults(func=folder_hash) + + # Scanoss settings options + for p in [p_folder_scan, p_scan, p_wfp, p_folder_hash]: + p.add_argument( + '--settings', + '-st', + type=str, + help='Settings file to use for scanning (optional - default scanoss.json)', + ) + p.add_argument( + '--skip-settings-file', + '-stf', + action='store_true', + help='Skip default settings file (scanoss.json) if it exists', + ) + for p in [p_copyleft, p_undeclared]: p.add_argument('-i', '--input', nargs='?', help='Path to results file') p.add_argument( @@ -559,7 +634,7 @@ def setup_args() -> None: # noqa: PLR0915 ) # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan]: p.add_argument( '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' ) @@ -570,10 +645,11 @@ def setup_args() -> None: # noqa: PLR0915 'Can also use the environment variable "grcp_proxy=:"', ) p.add_argument( - '--header','-hdr', + '--header', + '-hdr', action='append', # This allows multiple -H flags type=str, - help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times' + help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times', ) # Help/Trace command options @@ -594,7 +670,8 @@ def setup_args() -> None: # noqa: PLR0915 p_results, p_undeclared, p_copyleft, - c_provenance + c_provenance, + p_folder_scan, ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') @@ -607,12 +684,12 @@ def setup_args() -> None: # noqa: PLR0915 if not args.subparser: parser.print_help() # No sub command subcommand, print general help sys.exit(1) - elif ( - args.subparser in {'utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins'} and not args.subparsercmd): + elif (args.subparser in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins')) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed sys.exit(1) args.func(parser, args) # Execute the function associated with the sub-command + def ver(*_): """ Run the "ver" sub-command @@ -766,7 +843,7 @@ def get_scan_options(args): return scan_options -def scan(parser, args): # noqa: PLR0912, PLR0915 +def scan(parser, args): # noqa: PLR0912, PLR0915 """ Run the "scan" sub-command Parameters @@ -861,7 +938,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 if flags: print_stderr(f'Using flags {flags}...') elif not args.quiet: - if args.timeout < MIN_TIMEOUT_VALUE: + if args.timeout < MIN_TIMEOUT: print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.') if args.retry < 0: print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.') @@ -910,7 +987,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, - req_headers= process_req_headers(args.header), + req_headers=process_req_headers(args.header), ) if args.wfp: if not scanner.is_file_or_snippet_scan(): @@ -1122,7 +1199,7 @@ def utils_certloc(*_): print(f'CA Cert File: {certifi.where()}') -def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912 +def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912 """ Run the "utils cert-download" sub-command :param _: ignore/unused @@ -1149,13 +1226,14 @@ def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912 certs = conn.get_peer_cert_chain() for index, cert in enumerate(certs): cert_components = dict(cert.get_subject().get_components()) - if sys.version_info[0] >= PYTHON3_OR_LATER: + if sys.version_info[0] >= PYTHON_MAJOR_VERSION: cn = cert_components.get(b'CN') else: + # Fallback for Python versions less than PYTHON_MAJOR_VERSION cn = cert_components.get('CN') if not args.quiet: print_stderr(f'Certificate {index} - CN: {cn}') - if sys.version_info[0] >= PYTHON3_OR_LATER: + if sys.version_info[0] >= PYTHON_MAJOR_VERSION: print( (crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode('utf-8')).strip(), file=file ) # Print the downloaded PEM certificate @@ -1248,7 +1326,7 @@ def comp_crypto(parser, args): grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, - req_headers= process_req_headers(args.header), + req_headers=process_req_headers(args.header), ) if not comps.get_crypto_details(args.input, args.purl, args.output): sys.exit(1) @@ -1406,6 +1484,7 @@ def comp_versions(parser, args): if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit): sys.exit(1) + def comp_provenance(parser, args): """ Run the "component semgrep" sub-command @@ -1424,12 +1503,23 @@ def comp_provenance(parser, args): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') sys.exit(1) pac_file = get_pac_file(args.pac) - comps = Components(debug=args.debug, trace=args.trace, quiet=args.quiet, grpc_url=args.api2url, api_key=args.key, - ca_cert=args.ca_cert, proxy=args.proxy, grpc_proxy=args.grpc_proxy, pac=pac_file, - timeout=args.timeout, req_headers=process_req_headers(args.header)) + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.api2url, + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + timeout=args.timeout, + req_headers=process_req_headers(args.header), + ) if not comps.get_provenance_details(args.input, args.purl, args.output): sys.exit(1) + def results(parser, args): """ Run the "results" sub-command @@ -1488,13 +1578,99 @@ def process_req_headers(headers_array: List[str]) -> dict: dict_headers = {} for header_str in headers_array: # Split each "Name: Value" header - parts = header_str.split(":", 1) + parts = header_str.split(':', 1) if len(parts) == HEADER_PARTS_COUNT: name = parts[0].strip() value = parts[1].strip() dict_headers[name] = value return dict_headers + +def folder_hashing_scan(parser, args): + """Run the "folder-scan" sub-command + + Args: + parser (ArgumentParser): command line parser object + args (Namespace): Parsed arguments + """ + try: + if not args.scan_dir: + print_stderr('ERROR: Please specify a directory to scan') + parser.parse_args([args.subparser, '-h']) + sys.exit(1) + + if not os.path.exists(args.scan_dir) or not os.path.isdir(args.scan_dir): + print_stderr(f'ERROR: The specified directory {args.scan_dir} does not exist') + sys.exit(1) + + scanner_config = create_scanner_config_from_args(args) + scanoss_settings = get_scanoss_settings_from_args(args) + grpc_config = create_grpc_config_from_args(args) + + client = ScanossGrpc(**asdict(grpc_config)) + + scanner = ScannerHFH( + scan_dir=args.scan_dir, + config=scanner_config, + client=client, + scanoss_settings=scanoss_settings, + ) + + scanner.best_match = args.best_match + scanner.threshold = args.threshold + + scanner.scan() + scanner.present(output_file=args.output, output_format=args.format) + except ScanossGrpcError as e: + print_stderr(f'ERROR: {e}') + sys.exit(1) + + +def folder_hash(parser, args): + """Run the "folder-hash" sub-command + + Args: + parser (ArgumentParser): command line parser object + args (Namespace): Parsed arguments + """ + try: + if not args.scan_dir: + print_stderr('ERROR: Please specify a directory to scan') + parser.parse_args([args.subparser, '-h']) + sys.exit(1) + + if not os.path.exists(args.scan_dir) or not os.path.isdir(args.scan_dir): + print_stderr(f'ERROR: The specified directory {args.scan_dir} does not exist') + sys.exit(1) + + folder_hasher_config = create_folder_hasher_config_from_args(args) + scanoss_settings = get_scanoss_settings_from_args(args) + + folder_hasher = FolderHasher( + scan_dir=args.scan_dir, + config=folder_hasher_config, + scanoss_settings=scanoss_settings, + ) + + folder_hasher.hash_directory(args.scan_dir) + folder_hasher.present(output_file=args.output, output_format=args.format) + except Exception as e: + print_stderr(f'ERROR: {e}') + sys.exit(1) + + +def get_scanoss_settings_from_args(args): + scanoss_settings = None + if not args.skip_settings_file: + scanoss_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) + try: + scanoss_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new').set_scan_type('identify') + except ScanossSettingsError as e: + print_stderr(f'Error: {e}') + sys.exit(1) + return scanoss_settings + + def main(): """ Run the ScanOSS CLI diff --git a/src/scanoss/constants.py b/src/scanoss/constants.py new file mode 100644 index 00000000..3bce0ee9 --- /dev/null +++ b/src/scanoss/constants.py @@ -0,0 +1,12 @@ +DEFAULT_POST_SIZE = 32 +DEFAULT_TIMEOUT = 180 +DEFAULT_RETRY = 5 +MIN_TIMEOUT = 5 + +PYTHON_MAJOR_VERSION = 3 + +DEFAULT_SC_TIMEOUT = 600 +DEFAULT_NB_THREADS = 5 + +DEFAULT_URL = 'https://api.osskb.org' # default free service URL +DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index 85ee7ed7..f7c1950b 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -29,7 +29,6 @@ from pathspec import GitIgnoreSpec -from .scanoss_settings import ScanossSettings from .scanossbase import ScanossBase # Files to skip @@ -47,6 +46,19 @@ 'copying.lib', 'makefile', } + +DEFAULT_SKIPPED_FILES_HFH = { + 'gradlew', + 'gradlew.bat', + 'mvnw', + 'mvnw.cmd', + 'gradle-wrapper.jar', + 'maven-wrapper.jar', + 'thumbs.db', + 'babel.config.js', +} + + # Folders to skip DEFAULT_SKIPPED_DIRS = { 'nbproject', @@ -59,9 +71,34 @@ 'wheels', 'htmlcov', '__pypackages__', + 'example', + 'examples', + 'docs', + 'tests', + 'doc', + 'test', +} + +DEFAULT_SKIPPED_DIRS_HFH = { + 'nbproject', + 'nbbuild', + 'nbdist', + '__pycache__', + 'venv', + '_yardoc', + 'eggs', + 'wheels', + 'htmlcov', + '__pypackages__', + 'example', + 'examples', } + + # Folder endings to skip DEFAULT_SKIPPED_DIR_EXT = {'.egg-info'} +DEFAULT_SKIPPED_DIR_EXT_HFH = {'.egg-info'} + # File extensions to skip DEFAULT_SKIPPED_EXT = { '.1', @@ -205,6 +242,16 @@ '.gml', '.pot', '.plt', + '.whml', + '.pom', + '.smtml', + '.min.js', + '.mf', + '.base64', + '.s', + '.diff', + '.patch', + '.rules', # File endings '-doc', 'changelog', @@ -226,6 +273,162 @@ 'sqlite3', } +# TODO: For hfh add the .gitignore patterns +DEFAULT_SKIPPED_EXT_HFH = { + '.1', + '.2', + '.3', + '.4', + '.5', + '.6', + '.7', + '.8', + '.9', + '.ac', + '.adoc', + '.am', + '.asciidoc', + '.bmp', + '.build', + '.cfg', + '.chm', + '.class', + '.cmake', + '.cnf', + '.conf', + '.config', + '.contributors', + '.copying', + '.crt', + '.csproj', + '.css', + '.csv', + '.dat', + '.data', + '.dtd', + '.dts', + '.iws', + '.c9', + '.c9revisions', + '.dtsi', + '.dump', + '.eot', + '.eps', + '.geojson', + '.gif', + '.glif', + '.gmo', + '.guess', + '.hex', + '.htm', + '.html', + '.ico', + '.iml', + '.in', + '.inc', + '.info', + '.ini', + '.ipynb', + '.jpeg', + '.jpg', + '.json', + '.jsonld', + '.lock', + '.log', + '.m4', + '.map', + '.md5', + '.meta', + '.mk', + '.mxml', + '.o', + '.otf', + '.out', + '.pbtxt', + '.pdf', + '.pem', + '.phtml', + '.plist', + '.png', + '.prefs', + '.properties', + '.pyc', + '.qdoc', + '.result', + '.rgb', + '.rst', + '.scss', + '.sha', + '.sha1', + '.sha2', + '.sha256', + '.sln', + '.spec', + '.sub', + '.svg', + '.svn-base', + '.tab', + '.template', + '.test', + '.tex', + '.tiff', + '.ttf', + '.txt', + '.utf-8', + '.vim', + '.wav', + '.woff', + '.woff2', + '.xht', + '.xhtml', + '.xml', + '.xpm', + '.xsd', + '.xul', + '.yaml', + '.yml', + '.wfp', + '.editorconfig', + '.dotcover', + '.pid', + '.lcov', + '.egg', + '.manifest', + '.cache', + '.coverage', + '.cover', + '.gem', + '.lst', + '.pickle', + '.pdb', + '.gml', + '.pot', + '.plt', + '.whml', + '.pom', + '.smtml', + '.min.js', + '.mf', + '.base64', + '.s', + '.diff', + '.patch', + '.rules', + # File endings + '-doc', + 'config', + 'news', + 'readme', + 'swiftdoc', + 'texidoc', + 'todo', + 'version', + 'ignore', + 'manifest', + 'sqlite', + 'sqlite3', +} + class FileFilters(ScanossBase): """ @@ -233,20 +436,7 @@ class FileFilters(ScanossBase): Handles both inclusion and exclusion rules based on file paths, extensions, and sizes. """ - def __init__( - self, - debug: bool = False, - trace: bool = False, - quiet: bool = False, - scanoss_settings: 'ScanossSettings | None' = None, - all_extensions: bool = False, - all_folders: bool = False, - hidden_files_folders: bool = False, - operation_type: str = 'scanning', - skip_size: int = 0, - skip_extensions=None, - skip_folders=None, - ): + def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False, **kwargs): """ Initialize scan filters based on default settings. Optionally append custom settings. @@ -254,27 +444,29 @@ def __init__( debug (bool): Enable debug output trace (bool): Enable trace output quiet (bool): Suppress output - scanoss_settings (ScanossSettings): Custom settings to override defaults - all_extensions (bool): Include all file extensions - all_folders (bool): Include all folders - hidden_files_folders (bool): Include hidden files and folders - operation_type: operation type. can be either 'scanning' or 'fingerprinting' + **kwargs: Additional arguments including: + scanoss_settings (ScanossSettings): Custom settings to override defaults + all_extensions (bool): Include all file extensions + all_folders (bool): Include all folders + hidden_files_folders (bool): Include hidden files and folders + operation_type (str): Operation type ('scanning' or 'fingerprinting') + skip_size (int): Size to skip + skip_extensions (list): Extensions to skip + skip_folders (list): Folders to skip + is_folder_hashing_scan (bool): Whether the operation is a folder hashing scan """ super().__init__(debug, trace, quiet) - if skip_folders is None: - skip_folders = [] - if skip_extensions is None: - skip_extensions = [] - self.hidden_files_folders = hidden_files_folders - self.scanoss_settings = scanoss_settings - self.all_extensions = all_extensions - self.all_folders = all_folders - self.skip_folders = skip_folders - self.skip_size = skip_size - self.skip_extensions = skip_extensions - self.file_folder_pat_spec = self._get_file_folder_pattern_spec(operation_type) - self.size_pat_rules = self._get_size_limit_pattern_rules(operation_type) + self.hidden_files_folders = kwargs.get('hidden_files_folders', False) + self.scanoss_settings = kwargs.get('scanoss_settings') + self.all_extensions = kwargs.get('all_extensions', False) + self.all_folders = kwargs.get('all_folders', False) + self.skip_folders = kwargs.get('skip_folders', []) + self.skip_size = kwargs.get('skip_size', 0) + self.skip_extensions = kwargs.get('skip_extensions', []) + self.is_folder_hashing_scan = kwargs.get('is_folder_hashing_scan', False) + self.file_folder_pat_spec = self._get_file_folder_pattern_spec(kwargs.get('operation_type', 'scanning')) + self.size_pat_rules = self._get_size_limit_pattern_rules(kwargs.get('operation_type', 'scanning')) def get_filtered_files_from_folder(self, root: str) -> List[str]: """ @@ -304,16 +496,16 @@ def get_filtered_files_from_folder(self, root: str) -> List[str]: return all_files # Walk the tree looking for files to process. While taking into account files/folders to skip for dirpath, dirnames, filenames in os.walk(root_path): - dirpath = Path(dirpath) - rel_path = dirpath.relative_to(root_path) - if dirpath.is_symlink(): # TODO should we skip symlink folders? - self.print_msg(f'WARNING: Found symbolic link folder: {dirpath}') + dir_path = Path(dirpath) + rel_path = dir_path.relative_to(root_path) + if dir_path.is_symlink(): # TODO should we skip symlink folders? + self.print_msg(f'WARNING: Found symbolic link folder: {dir_path}') - if self._should_skip_dir(str(rel_path)): # Current directory should be skipped + if self.should_skip_dir(str(rel_path)): # Current directory should be skipped dirnames.clear() continue for filename in filenames: - file_path = dirpath / filename + file_path = dir_path / filename all_files.append(str(file_path)) # End os.walk loop # Now filter the files and return the reduced list @@ -332,30 +524,36 @@ def get_filtered_files_from_files(self, files: List[str], scan_root: str = None) """ filtered_files = [] for file_path in files: - if not os.path.exists(file_path) or not os.path.isfile(file_path) or os.path.islink(file_path): - self.print_debug( - f'WARNING: File {file_path} does not exist, is not a file, or is a symbolic link. Ignoring.' - ) - continue + path_obj = Path(file_path) try: if scan_root: - rel_path = os.path.relpath(file_path, scan_root) + rel_path = path_obj.relative_to(scan_root) else: - rel_path = os.path.relpath(file_path) + rel_path = str(path_obj) except ValueError: - # If file_path is broken, symlink ignore it self.print_debug(f'Ignoring file: {file_path} (broken symlink)') continue + + if not path_obj.exists() or not path_obj.is_file() or path_obj.is_symlink(): + self.print_debug( + f'WARNING: File {rel_path} does not exist, is not a file, or is a symbolic link. Ignoring.' + ) + continue + + if not self.hidden_files_folders and any(part.startswith('.') for part in path_obj.parts): + self.print_debug(f'Skipping file: {rel_path} (in hidden directory or is hidden file)') + continue + if self._should_skip_file(rel_path): continue try: - file_size = os.path.getsize(file_path) + file_size = path_obj.stat().st_size if file_size == 0: self.print_debug(f'Skipping file: {rel_path} (empty file)') continue min_size, max_size = self._get_operation_size_limits(file_path) if min_size <= file_size <= max_size: - filtered_files.append(rel_path) + filtered_files.append(str(rel_path)) else: self.print_debug( f'Skipping file: {rel_path} (size {file_size} outside limits {min_size}-{max_size})' @@ -369,8 +567,11 @@ def _get_file_folder_pattern_spec(self, operation_type: str = 'scanning'): """ Get file path pattern specification. - :param operation_type: which operation is being performed - :return: List of file path patterns + Args: + operation_type (str): Type of operation ('scanning' or 'fingerprinting') + + Returns: + GitIgnoreSpec: GitIgnoreSpec object containing the file path patterns """ patterns = self._get_operation_patterns(operation_type) if patterns: @@ -381,8 +582,11 @@ def _get_size_limit_pattern_rules(self, operation_type: str = 'scanning'): """ Get size limit pattern rules. - :param operation_type: which operation is being performed - :return: List of size limit pattern rules + Args: + operation_type (str): Type of operation ('scanning' or 'fingerprinting') + + Returns: + List of size limit pattern rules """ if self.scanoss_settings: size_rules = self.scanoss_settings.get_skip_sizes(operation_type) @@ -407,6 +611,14 @@ def _get_operation_patterns(self, operation_type: str) -> List[str]: List[str]: Combined list of patterns to skip """ patterns = [] + + # Default patterns for skipping directories + if not self.all_folders: + DEFAULT_SKIPPED_DIR_LIST = DEFAULT_SKIPPED_DIRS_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_DIRS + for dir_name in DEFAULT_SKIPPED_DIR_LIST: + patterns.append(f'{dir_name}/') + + # Custom patterns added in SCANOSS settings file if self.scanoss_settings: patterns.extend(self.scanoss_settings.get_skip_patterns(operation_type)) return patterns @@ -445,7 +657,7 @@ def _get_operation_size_limits(self, file_path: str = None) -> tuple: # End rules loop return min_size, max_size - def _should_skip_dir(self, dir_rel_path: str) -> bool: + def should_skip_dir(self, dir_rel_path: str) -> bool: # noqa: PLR0911 """ Check if a directory should be skipped based on operation type and default rules. @@ -483,7 +695,7 @@ def _should_skip_dir(self, dir_rel_path: str) -> bool: return True return False - def _should_skip_file(self, file_rel_path: str) -> bool: + def _should_skip_file(self, file_rel_path: str) -> bool: # noqa: PLR0911 """ Check if a file should be skipped based on operation type and default rules. @@ -495,6 +707,9 @@ def _should_skip_file(self, file_rel_path: str) -> bool: """ file_name = os.path.basename(file_rel_path) + DEFAULT_SKIPPED_FILES_LIST = DEFAULT_SKIPPED_FILES_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_FILES + DEFAULT_SKIPPED_EXT_LIST = DEFAULT_SKIPPED_EXT_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_EXT + if not self.hidden_files_folders and file_name.startswith('.'): self.print_debug(f'Skipping file: {file_rel_path} (hidden file)') return True @@ -502,11 +717,11 @@ def _should_skip_file(self, file_rel_path: str) -> bool: return False file_name_lower = file_name.lower() # Look for exact files - if file_name_lower in DEFAULT_SKIPPED_FILES: + if file_name_lower in DEFAULT_SKIPPED_FILES_LIST: self.print_debug(f'Skipping file: {file_rel_path} (matches default skip file)') return True # Look for file endings - for ending in DEFAULT_SKIPPED_EXT: + for ending in DEFAULT_SKIPPED_EXT_LIST: if file_name_lower.endswith(ending): self.print_debug(f'Skipping file: {file_rel_path} (matches default skip ending: {ending})') return True diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 8ca80aa3..5354b124 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -25,6 +25,8 @@ import json from typing import Any, Dict, List +from scanoss.utils.abstract_presenter import AbstractPresenter + from .scanossbase import ScanossBase MATCH_TYPES = ['file', 'snippet'] @@ -47,16 +49,77 @@ 'status': ['pending'], } -AVAILABLE_OUTPUT_FORMATS = ['json', 'plain'] +class ResultsPresenter(AbstractPresenter): + """ + SCANOSS Results presenter class + Handles the presentation of the scan results + """ + + def __init__(self, results_instance, **kwargs): + super().__init__(**kwargs) + self.results = results_instance + + def _format_json_output(self) -> str: + """ + Format the output data into a JSON object + """ + + formatted_data = [] + for item in self.results.data: + formatted_data.append( + { + 'file': item.get('filename'), + 'status': item.get('status', 'N/A'), + 'match_type': item['id'], + 'matched': item.get('matched', 'N/A'), + 'purl': (item.get('purl')[0] if item.get('purl') else 'N/A'), + 'license': (item.get('licenses')[0].get('name', 'N/A') if item.get('licenses') else 'N/A'), + } + ) + try: + return json.dumps({'results': formatted_data, 'total': len(formatted_data)}, indent=2) + except Exception as e: + self.base.print_stderr(f'ERROR: Problem formatting JSON output: {e}') + return '' + + def _format_plain_output(self) -> str: + """Format the output data into a plain text string + + Returns: + str: The formatted output data + """ + if not self.results.data: + msg = 'No results to present' + return msg + + formatted = '' + for item in self.results.data: + formatted += f'{self._format_plain_output_item(item)}\n' + return formatted -class Results(ScanossBase): + @staticmethod + def _format_plain_output_item(item): + purls = item.get('purl', []) + licenses = item.get('licenses', []) + + return ( + f'File: {item.get("filename")}\n' + f'Match type: {item.get("id")}\n' + f'Status: {item.get("status", "N/A")}\n' + f'Matched: {item.get("matched", "N/A")}\n' + f'Purl: {purls[0] if purls else "N/A"}\n' + f'License: {licenses[0].get("name", "N/A") if licenses else "N/A"}\n' + ) + + +class Results: """ SCANOSS Results class \n Handles the parsing and filtering of the scan results """ - def __init__( + def __init__( # noqa: PLR0913 self, debug: bool = False, trace: bool = False, @@ -80,11 +143,17 @@ def __init__( output_format (str, optional): Output format. Defaults to None. """ - super().__init__(debug, trace, quiet) + self.base = ScanossBase(debug, trace, quiet) self.data = self._load_and_transform(filepath) self.filters = self._load_filters(match_type=match_type, status=status) - self.output_file = output_file - self.output_format = output_format + self.presenter = ResultsPresenter( + self, + debug=debug, + trace=trace, + quiet=quiet, + output_file=output_file, + output_format=output_format, + ) def load_file(self, file: str) -> Dict[str, Any]: """Load the JSON file @@ -99,7 +168,7 @@ def load_file(self, file: str) -> Dict[str, Any]: try: return json.load(jsonfile) except Exception as e: - self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + self.base.print_stderr(f'ERROR: Problem parsing input JSON: {e}') def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: """ @@ -174,8 +243,8 @@ def _item_matches_filters(self, item): def _validate_filter_values(filter_key: str, filter_value: List[str]): if any(value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) for value in filter_value): valid_values = ', '.join(AVAILABLE_FILTER_VALUES.get(filter_key, [])) - raise Exception( - f"ERROR: Invalid filter value '{filter_value}' for filter '{filter_key.value}'. " + raise ValueError( + f"ERROR: Invalid filter value '{filter_value}' for filter '{filter_key}'. " f'Valid values are: {valid_values}' ) @@ -190,103 +259,5 @@ def has_results(self): return bool(self.data) def present(self, output_format: str = None, output_file: str = None): - """Format and present the results. If no output format is provided, the results will be printed to stdout - - Args: - output_format (str, optional): Output format. Defaults to None. - output_file (str, optional): Output file. Defaults to None. - - Raises: - Exception: Invalid output format - - Returns: - None - """ - file_path = output_file or self.output_file - fmt = output_format or self.output_format - - if fmt and fmt not in AVAILABLE_OUTPUT_FORMATS: - raise Exception( - f"ERROR: Invalid output format '{output_format}'. Valid values are: {', '.join(AVAILABLE_OUTPUT_FORMATS)}" - ) - - if fmt == 'json': - return self._present_json(file_path) - elif fmt == 'plain': - return self._present_plain(file_path) - else: - return self._present_stdout() - - def _present_json(self, file: str = None): - """Present the results in JSON format - - Args: - file (str, optional): Output file. Defaults to None. - """ - self.print_to_file_or_stdout(json.dumps(self._format_json_output(), indent=2), file) - - def _format_json_output(self): - """ - Format the output data into a JSON object - """ - - formatted_data = [] - for item in self.data: - formatted_data.append( - { - 'file': item.get('filename'), - 'status': item.get('status', 'N/A'), - 'match_type': item['id'], - 'matched': item.get('matched', 'N/A'), - 'purl': (item.get('purl')[0] if item.get('purl') else 'N/A'), - 'license': (item.get('licenses')[0].get('name', 'N/A') if item.get('licenses') else 'N/A'), - } - ) - return {'results': formatted_data, 'total': len(formatted_data)} - - def _present_plain(self, file: str = None): - """Present the results in plain text format - - Args: - file (str, optional): Output file. Defaults to None. - - Returns: - None - """ - if not self.data: - return self.print_stderr('No results to present') - self.print_to_file_or_stdout(self._format_plain_output(), file) - - def _present_stdout(self): - """Present the results to stdout - - Returns: - None - """ - if not self.data: - return self.print_stderr('No results to present') - self.print_to_file_or_stdout(self._format_plain_output()) - - def _format_plain_output(self): - """ - Format the output data into a plain text string - """ - - formatted = '' - for item in self.data: - formatted += f'{self._format_plain_output_item(item)} \n' - return formatted - - @staticmethod - def _format_plain_output_item(item): - purls = item.get('purl', []) - licenses = item.get('licenses', []) - - return ( - f'File: {item.get("filename")}\n' - f'Match type: {item.get("id")}\n' - f'Status: {item.get("status", "N/A")}\n' - f'Matched: {item.get("matched", "N/A")}\n' - f'Purl: {purls[0] if purls else "N/A"}\n' - f'License: {licenses[0].get("name", "N/A") if licenses else "N/A"}\n' - ) + """Present the results in the selected format""" + self.presenter.present(output_format=output_format, output_file=output_file) diff --git a/src/scanoss/scanners/__init__.py b/src/scanoss/scanners/__init__.py new file mode 100644 index 00000000..1e95c46d --- /dev/null +++ b/src/scanoss/scanners/__init__.py @@ -0,0 +1,23 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py new file mode 100644 index 00000000..6d4e4af7 --- /dev/null +++ b/src/scanoss/scanners/folder_hasher.py @@ -0,0 +1,290 @@ +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Literal, Optional + +from progress.bar import Bar + +from scanoss.file_filters import FileFilters +from scanoss.scanoss_settings import ScanossSettings +from scanoss.scanossbase import ScanossBase +from scanoss.utils.abstract_presenter import AbstractPresenter +from scanoss.utils.crc64 import CRC64 +from scanoss.utils.simhash import WordFeatureSet, fingerprint, simhash, vectorize_bytes + +MINIMUM_FILE_COUNT = 8 +MINIMUM_CONCATENATED_NAME_LENGTH = 32 +MINIMUM_FILE_NAME_LENGTH = 32 + + +class DirectoryNode: + """ + Represents a node in the directory tree for folder hashing. + """ + + def __init__(self, path: str): + self.path = path + self.is_dir = True + self.children: Dict[str, DirectoryNode] = {} + self.files: List[DirectoryFile] = [] + + +class DirectoryFile: + """ + Represents a file in the directory tree for folder hashing. + """ + + def __init__(self, path: str, key: bytes, key_str: str): + self.path = path + self.key = key + self.key_str = key_str + + +@dataclass +class FolderHasherConfig: + debug: bool = False + trace: bool = False + quiet: bool = False + output_file: Optional[str] = None + output_format: Literal['json'] = 'json' + settings_file: Optional[str] = None + skip_settings_file: bool = False + + +def create_folder_hasher_config_from_args(args) -> FolderHasherConfig: + return FolderHasherConfig( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + output_file=getattr(args, 'output', None), + output_format=getattr(args, 'format', 'json'), + settings_file=getattr(args, 'settings', None), + skip_settings_file=getattr(args, 'skip_settings_file', False), + ) + + +class FolderHasher: + """ + Folder Hasher. + + This class is used to produce a folder hash for a given directory. + + It builds a directory tree (DirectoryNode) and computes the associated + hash data for the folder. + """ + + def __init__( + self, + scan_dir: str, + config: Optional[FolderHasherConfig] = None, + scanoss_settings: Optional[ScanossSettings] = None, + ): + self.base = ScanossBase( + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + self.file_filters = FileFilters( + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + scanoss_settings=scanoss_settings, + is_folder_hashing_scan=True, + ) + self.presenter = FolderHasherPresenter( + self, + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + + self.scan_dir = scan_dir + self.tree = None + + def hash_directory(self, path: str) -> dict: + """ + Generate the folder hashing request structure from a directory path. + + This method builds a directory tree (DirectoryNode) and computes the associated + hash data for the folder. + + Args: + path (str): The root directory path. + + Returns: + dict: The folder hash request structure. + """ + + root_node = self._build_root_node(path) + tree = self._hash_calc_from_node(root_node) + + self.tree = tree + + return tree + + def _build_root_node(self, path: str) -> DirectoryNode: + """ + Build a directory tree from the given path with file information. + + The tree includes DirectoryNode objects populated with filtered file items, + each containing their relative path and CRC64 hash key. + + Args: + path (str): The directory path to build the tree from. + + Returns: + DirectoryNode: The root node representing the directory. + """ + root = Path(path).resolve() + root_node = DirectoryNode(str(root)) + + all_files = [ + f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) < MINIMUM_FILE_NAME_LENGTH + ] + filtered_files = self.file_filters.get_filtered_files_from_files(all_files, str(root)) + + # Sort the files by name to ensure the hash is the same for the same folder + filtered_files.sort() + + bar = Bar('Hashing files...', max=len(filtered_files)) + for file_path in filtered_files: + try: + file_path_obj = Path(file_path) if isinstance(file_path, str) else file_path + full_file_path = file_path_obj if file_path_obj.is_absolute() else root / file_path_obj + + self.base.print_debug(f'\nHashing file {str(full_file_path)}') + + file_bytes = full_file_path.read_bytes() + key = CRC64.get_hash_buff(file_bytes) + key_str = ''.join(f'{b:02x}' for b in key) + rel_path = str(full_file_path.relative_to(root)) + + file_item = DirectoryFile(rel_path, key, key_str) + + current_node = root_node + for part in Path(rel_path).parent.parts: + child_path = str(Path(current_node.path) / part) + if child_path not in current_node.children: + current_node.children[child_path] = DirectoryNode(child_path) + current_node = current_node.children[child_path] + current_node.files.append(file_item) + + root_node.files.append(file_item) + + except Exception as e: + self.base.print_debug(f'Skipping file {full_file_path}: {str(e)}') + + bar.next() + + bar.finish() + return root_node + + def _hash_calc_from_node(self, node: DirectoryNode) -> dict: + """ + Recursively compute folder hash data for a directory node. + + The hash data includes the path identifier, simhash for file names, + simhash for file content, and children node hash information. + + Args: + node (DirectoryNode): The directory node to compute the hash for. + + Returns: + dict: The computed hash data for the node. + """ + hash_data = self._hash_calc(node) + + return { + 'path_id': node.path, + 'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None, + 'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None, + 'children': [self._hash_calc_from_node(child) for child in node.children.values()], + } + + def _hash_calc(self, node: DirectoryNode) -> dict: + """ + Compute folder hash values for a given directory node. + + The method aggregates unique file keys and sorted file names to generate + simhash-based hash values for both file names and file contents. + + The most significant byte of the name simhash is then replaced by a computed head value. + + Args: + node (DirectoryNode): The directory node containing file items. + + Returns: + dict: A dictionary with 'name_hash' and 'content_hash' keys. + """ + processed_hashes = set() + file_hashes = [] + selected_names = [] + + for file in node.files: + key_str = file.key_str + if key_str in processed_hashes: + continue + processed_hashes.add(key_str) + + selected_names.append(os.path.basename(file.path)) + + file_key = bytes(file.key) + file_hashes.append(file_key) + + if len(selected_names) < MINIMUM_FILE_COUNT: + return { + 'name_hash': None, + 'content_hash': None, + } + + selected_names.sort() + concatenated_names = ''.join(selected_names) + + if len(concatenated_names.encode('utf-8')) < MINIMUM_CONCATENATED_NAME_LENGTH: + return { + 'name_hash': None, + 'content_hash': None, + } + + names_simhash = simhash(WordFeatureSet(concatenated_names.encode('utf-8'))) + content_simhash = fingerprint(vectorize_bytes(file_hashes)) + + return { + 'name_hash': names_simhash, + 'content_hash': content_simhash, + } + + def present(self, output_format: str = None, output_file: str = None): + """Present the hashed tree in the selected format""" + self.presenter.present(output_format=output_format, output_file=output_file) + + +class FolderHasherPresenter(AbstractPresenter): + """ + FolderHasher presenter class + Handles the presentation of the folder hashing scan results + """ + + def __init__(self, folder_hasher: FolderHasher, **kwargs): + super().__init__(**kwargs) + self.folder_hasher = folder_hasher + + def _format_json_output(self) -> str: + """ + Format the scan output data into a JSON object + + Returns: + str: The formatted JSON string + """ + return json.dumps(self.folder_hasher.tree, indent=2) + + def _format_plain_output(self) -> str: + """ + Format the scan output data into a plain text string + """ + return ( + json.dumps(self.folder_hasher.tree, indent=2) + if isinstance(self.folder_hasher.tree, dict) + else str(self.folder_hasher.tree) + ) diff --git a/src/scanoss/scanners/scanner_config.py b/src/scanoss/scanners/scanner_config.py new file mode 100644 index 00000000..ed48342d --- /dev/null +++ b/src/scanoss/scanners/scanner_config.py @@ -0,0 +1,73 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +from dataclasses import dataclass +from typing import Optional + +from pypac.parser import PACFile + +from scanoss.constants import ( + DEFAULT_NB_THREADS, + DEFAULT_POST_SIZE, + DEFAULT_SC_TIMEOUT, + DEFAULT_TIMEOUT, +) + + +@dataclass +class ScannerConfig: + debug: bool = False + trace: bool = False + quiet: bool = False + api_key: Optional[str] = None + url: Optional[str] = None + grpc_url: Optional[str] = None + post_size: int = DEFAULT_POST_SIZE + timeout: int = DEFAULT_TIMEOUT + sc_timeout: int = DEFAULT_SC_TIMEOUT + nb_threads: int = DEFAULT_NB_THREADS + proxy: Optional[str] = None + grpc_proxy: Optional[str] = None + + ca_cert: Optional[str] = None + pac: Optional[PACFile] = None + + +def create_scanner_config_from_args(args) -> ScannerConfig: + return ScannerConfig( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + api_key=getattr(args, 'key', None), + url=getattr(args, 'api_url', None), + grpc_url=getattr(args, 'grpc_url', None), + post_size=getattr(args, 'post_size', DEFAULT_POST_SIZE), + timeout=getattr(args, 'timeout', DEFAULT_TIMEOUT), + sc_timeout=getattr(args, 'sc_timeout', DEFAULT_SC_TIMEOUT), + nb_threads=getattr(args, 'nb_threads', DEFAULT_NB_THREADS), + proxy=getattr(args, 'proxy', None), + grpc_proxy=getattr(args, 'grpc_proxy', None), + ca_cert=getattr(args, 'ca_cert', None), + pac=getattr(args, 'pac', None), + ) diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py new file mode 100644 index 00000000..0af82231 --- /dev/null +++ b/src/scanoss/scanners/scanner_hfh.py @@ -0,0 +1,160 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import threading +import time +from typing import Dict, Optional + +from progress.spinner import Spinner + +from scanoss.file_filters import FileFilters +from scanoss.scanners.folder_hasher import FolderHasher +from scanoss.scanners.scanner_config import ScannerConfig +from scanoss.scanoss_settings import ScanossSettings +from scanoss.scanossbase import ScanossBase +from scanoss.scanossgrpc import ScanossGrpc +from scanoss.utils.abstract_presenter import AbstractPresenter + + +class ScannerHFH: + """ + Folder Hashing Scanner. + + This scanner processes a directory, computes CRC64 hashes for the files, + and calculates simhash values based on file names and content to detect folder-level similarities. + """ + + def __init__( + self, + scan_dir: str, + config: ScannerConfig, + client: Optional[ScanossGrpc] = None, + scanoss_settings: Optional[ScanossSettings] = None, + ): + """ + Initialize the ScannerHFH. + + Args: + scan_dir (str): The directory to be scanned. + config (ScannerConfig): Configuration parameters for the scanner. + client (ScanossGrpc): gRPC client for communicating with the scanning service. + scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss. + """ + self.base = ScanossBase( + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + self.presenter = ScannerHFHPresenter( + self, + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + self.file_filters = FileFilters( + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + scanoss_settings=scanoss_settings, + ) + self.folder_hasher = FolderHasher( + scan_dir=scan_dir, + config=config, + scanoss_settings=scanoss_settings, + ) + + self.scan_dir = scan_dir + self.client = client + self.scan_results = None + self.best_match = False + self.threshold = 100 + + def scan(self) -> Optional[Dict]: + """ + Scan the provided directory using the folder hashing algorithm. + + Returns: + Optional[Dict]: The folder hash response from the gRPC client, or None if an error occurs. + """ + hfh_request = { + 'root': self.folder_hasher.hash_directory(self.scan_dir), + 'threshold': self.threshold, + 'best_match': self.best_match, + } + + spinner = Spinner('Scanning folder...') + stop_spinner = False + + def spin(): + while not stop_spinner: + spinner.next() + time.sleep(0.1) + + spinner_thread = threading.Thread(target=spin) + spinner_thread.start() + + try: + response = self.client.folder_hash_scan(hfh_request) + self.scan_results = response + finally: + stop_spinner = True + spinner_thread.join() + spinner.finish() + + return self.scan_results + + def present(self, output_format: str = None, output_file: str = None): + """Present the results in the selected format""" + self.presenter.present(output_format=output_format, output_file=output_file) + + +class ScannerHFHPresenter(AbstractPresenter): + """ + ScannerHFH presenter class + Handles the presentation of the folder hashing scan results + """ + + def __init__(self, scanner: ScannerHFH, **kwargs): + super().__init__(**kwargs) + self.scanner = scanner + + def _format_json_output(self) -> str: + """ + Format the scan output data into a JSON object + + Returns: + str: The formatted JSON string + """ + return json.dumps(self.scanner.scan_results, indent=2) + + def _format_plain_output(self) -> str: + """ + Format the scan output data into a plain text string + """ + return ( + json.dumps(self.scanner.scan_results, indent=2) + if isinstance(self.scanner.scan_results, dict) + else str(self.scanner.scan_results) + ) diff --git a/src/scanoss/scanossbase.py b/src/scanoss/scanossbase.py index c5f79c4a..64db54cc 100644 --- a/src/scanoss/scanossbase.py +++ b/src/scanoss/scanossbase.py @@ -80,15 +80,15 @@ def print_stdout(*args, **kwargs): **kwargs, ) - def print_to_file_or_stdout(self, msg: str, file: str = None): + def print_to_file_or_stdout(self, content: str, file: str = None): """ Print message to file if provided or stdout """ if file: with open(file, 'w') as f: - f.write(msg) + f.write(content) else: - self.print_stdout(msg) + self.print_stdout(content) def print_to_file_or_stderr(self, msg: str, file: str = None): """ diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 77f55501..6fb5569d 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -26,6 +26,9 @@ import json import os import uuid +from dataclasses import dataclass +from enum import IntEnum +from typing import Dict, Optional from urllib.parse import urlparse import grpc @@ -33,6 +36,10 @@ from pypac.parser import PACFile from pypac.resolver import ProxyResolver +from scanoss.api.provenance.v2.scanoss_provenance_pb2_grpc import ProvenanceStub +from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub +from scanoss.constants import DEFAULT_TIMEOUT + from . import __version__ from .api.common.v2.scanoss_common_pb2 import ( EchoRequest, @@ -50,10 +57,13 @@ from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub -from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest +from .api.dependencies.v2.scanoss_dependencies_pb2 import ( + DependencyRequest, + DependencyResponse, +) from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse -from .api.provenance.v2.scanoss_provenance_pb2_grpc import ProvenanceStub +from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse @@ -67,6 +77,22 @@ MAX_CONCURRENT_REQUESTS = 5 +class ScanossGrpcError(Exception): + """ + Custom exception for SCANOSS gRPC errors + """ + + pass + + +class ScanossGrpcStatusCode(IntEnum): + """Status codes for SCANOSS gRPC responses""" + + SUCCESS = 1 + SUCCESS_WITH_WARNINGS = 2 + FAILED_WITH_WARNINGS = 3 + FAILED = 4 + class ScanossGrpc(ScanossBase): """ @@ -147,6 +173,7 @@ def __init__( # noqa: PLR0913, PLR0915 self.semgrep_stub = SemgrepStub(grpc.insecure_channel(self.url)) self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url)) self.provenance_stub = ProvenanceStub(grpc.insecure_channel(self.url)) + self.scanning_stub = ScanningStub(grpc.insecure_channel(self.url)) else: if ca_cert is not None: credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate @@ -158,6 +185,7 @@ def __init__( # noqa: PLR0913, PLR0915 self.semgrep_stub = SemgrepStub(grpc.secure_channel(self.url, credentials)) self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials)) self.provenance_stub = ProvenanceStub(grpc.secure_channel(self.url, credentials)) + self.scanning_stub = ScanningStub(grpc.secure_channel(self.url, credentials)) @classmethod def _load_cert(cls, cert_file: str) -> bytes: @@ -437,6 +465,62 @@ def get_component_versions_json(self, search: dict) -> dict: return resp_dict return None + def folder_hash_scan(self, request: Dict) -> Dict: + """ + Client function to call the rpc for Folder Hashing Scan + + Args: + request (Dict): Folder Hash Request + + Returns: + Dict: Folder Hash Response + """ + return self._call_rpc( + self.scanning_stub.FolderHashScan, + request, + HFHRequest, + 'Sending folder hash scan data (rqId: {rqId})...', + ) + + def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional[str] = None) -> dict: + """ + Call a gRPC method and return the response as a dictionary + + Args: + rpc_method (): The gRPC stub method + request_input (): Either a dict or a gRPC request object. + request_type (): The type of the gRPC request object. + debug_msg (str, optional): Debug message template that can include {rqId} placeholder. + + Returns: + dict: The parsed gRPC response as a dictionary, or None if an error occurred. + """ + + request_id = str(uuid.uuid4()) + + if isinstance(request_input, dict): + request_obj = ParseDict(request_input, request_type()) + else: + request_obj = request_input + + metadata = self.metadata[:] + [('x-request-id', request_id)] + + self.print_debug(debug_msg.format(rqId=request_id)) + + try: + resp = rpc_method(request_obj, metadata=metadata, timeout=self.timeout) + except grpc.RpcError as e: + raise ScanossGrpcError( + f'{e.__class__.__name__} while sending gRPC message (rqId: {request_id}): {e.details()}' + ) + + if resp and not self._check_status_response(resp.status, request_id): + raise ScanossGrpcError(f'Unsuccessful status response (rqId: {request_id}).') + + resp_dict = MessageToDict(resp, preserving_proto_field_name=True) + resp_dict.pop('status', None) + return resp_dict + def _check_status_response(self, status_response: StatusResponse, request_id: str = None) -> bool: """ Check the response object to see if the command was successful or not @@ -452,13 +536,13 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st return True self.print_debug(f'Checking response status (rqId: {request_id}): {status_response}') status_code: StatusCode = status_response.status - if status_code > 1: + if status_code > ScanossGrpcStatusCode.SUCCESS: ret_val = False # default to failed msg = 'Unsuccessful' - if status_code == SUCCEDED_WITH_WARNINGS_STATUS_CODE: + if status_code == ScanossGrpcStatusCode.SUCCESS_WITH_WARNINGS: msg = 'Succeeded with warnings' ret_val = True # No need to fail as it succeeded with warnings - elif status_code == FAILED_STATUS_CODE: + elif status_code == ScanossGrpcStatusCode.FAILED_WITH_WARNINGS: msg = 'Failed with warnings' self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}') return ret_val @@ -532,3 +616,34 @@ def load_generic_headers(self): # # End of ScanossGrpc Class # + + +@dataclass +class GrpcConfig: + url: str = DEFAULT_URL + api_key: Optional[str] = SCANOSS_API_KEY + debug: Optional[bool] = False + trace: Optional[bool] = False + quiet: Optional[bool] = False + ver_details: Optional[str] = None + ca_cert: Optional[str] = None + pac: Optional[PACFile] = None + timeout: Optional[int] = DEFAULT_TIMEOUT + proxy: Optional[str] = None + grpc_proxy: Optional[str] = None + + +def create_grpc_config_from_args(args) -> GrpcConfig: + return GrpcConfig( + url=getattr(args, 'api2url', DEFAULT_URL), + api_key=getattr(args, 'key', SCANOSS_API_KEY), + debug=getattr(args, 'debug', False), + trace=getattr(args, 'trace', False), + quiet=getattr(args, 'quiet', False), + ver_details=getattr(args, 'ver_details', None), + ca_cert=getattr(args, 'ca_cert', None), + pac=getattr(args, 'pac', None), + timeout=getattr(args, 'timeout', DEFAULT_TIMEOUT), + proxy=getattr(args, 'proxy', None), + grpc_proxy=getattr(args, 'grpc_proxy', None), + ) diff --git a/src/scanoss/utils/abstract_presenter.py b/src/scanoss/utils/abstract_presenter.py new file mode 100644 index 00000000..bbc591ee --- /dev/null +++ b/src/scanoss/utils/abstract_presenter.py @@ -0,0 +1,68 @@ +from abc import ABC, abstractmethod + +from scanoss.scanossbase import ScanossBase + +AVAILABLE_OUTPUT_FORMATS = ['json', 'plain'] + + +class AbstractPresenter(ABC): + """ + Abstract presenter class for presenting output in a given format. + Subclasses must implement the _format_json_output and _format_plain_output methods. + """ + + def __init__( + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + output_file: str = None, + output_format: str = None, + ): + """ + Initialize the presenter with the given output file and format. + """ + self.base = ScanossBase(debug=debug, trace=trace, quiet=quiet) + self.output_file = output_file + self.output_format = output_format + + def present(self, output_format: str = None, output_file: str = None): + """ + Present the formatted output to a file if provided; otherwise, print to stdout. + """ + file_path = output_file or self.output_file + fmt = output_format or self.output_format + + if fmt and fmt not in AVAILABLE_OUTPUT_FORMATS: + raise ValueError( + f"ERROR: Invalid output format '{fmt}'. Valid values are: {', '.join(AVAILABLE_OUTPUT_FORMATS)}" + ) + + if fmt == 'json': + content = self._format_json_output() + elif fmt == 'plain': + content = self._format_plain_output() + else: + content = self._format_plain_output() + + self._present_output(content, file_path) + + def _present_output(self, content: str, file_path: str = None): + """ + If a file path is provided, write to that file; otherwise, print the content to stdout. + """ + self.base.print_to_file_or_stdout(content, file_path) + + @abstractmethod + def _format_json_output(self) -> str: + """ + Return a JSON string representation of the data. + """ + pass + + @abstractmethod + def _format_plain_output(self) -> str: + """ + Return a plain text string representation of the data. + """ + pass diff --git a/src/scanoss/utils/crc64.py b/src/scanoss/utils/crc64.py new file mode 100644 index 00000000..d39785d6 --- /dev/null +++ b/src/scanoss/utils/crc64.py @@ -0,0 +1,96 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import struct +from typing import List + + +class CRC64: + """ + CRC64 ECMA implementation matching Go's hash/crc64 package. + Uses polynomial: 0xC96C5795D7870F42 + """ + + POLY = 0xC96C5795D7870F42 + _TABLE = None + + def __init__(self): + if CRC64._TABLE is None: + CRC64._TABLE = self._make_table() + self.crc = 0xFFFFFFFFFFFFFFFF # Initial value + + def _make_table(self) -> list: + """Generate the CRC64 lookup table.""" + table = [] + for i in range(256): + crc = i + for _ in range(8): + if crc & 1: + crc = (crc >> 1) ^ self.POLY + else: + crc >>= 1 + table.append(crc) + return table + + def update(self, data: bytes) -> None: + """Update the CRC with new data.""" + if isinstance(data, str): + data = data.encode('utf-8') + + crc = self.crc + for b in data: + crc = (crc >> 8) ^ CRC64._TABLE[(crc ^ b) & 0xFF] # Use class-level table + self.crc = crc + + def digest(self) -> int: + """Get the current CRC value.""" + return self.crc ^ 0xFFFFFFFFFFFFFFFF # Final XOR value + + def hexdigest(self): + """Get the current CRC value as a hexadecimal string.""" + return format(self.digest(), '016x') + + @classmethod + def checksum(cls, data: bytes) -> int: + """Calculate CRC64 checksum for the given data.""" + crc = cls() + crc.update(data) + return crc.digest() + + @classmethod + def get_hash_buff(cls, buff: bytes) -> List[bytes]: + """ + Get the hash value of the given buffer, and converts it to 8 bytes in big-endian order. + + Args: + buff (bytes): The buffer to get the hash value of. + + Returns: + bytes: The hash value of the given buffer, and converts it to 8 bytes in big-endian order. + """ + crc = cls() + crc.update(buff) + hash_val = crc.digest() + + return list(struct.pack('>Q', hash_val)) diff --git a/src/scanoss/utils/simhash.py b/src/scanoss/utils/simhash.py new file mode 100644 index 00000000..1ed56886 --- /dev/null +++ b/src/scanoss/utils/simhash.py @@ -0,0 +1,198 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import re +import unicodedata + +FNV64_OFFSET_BASIS = 14695981039346656037 +FNV64_PRIME = 1099511628211 +MASK64 = 0xFFFFFFFFFFFFFFFF + + +def fnv1_64(data: bytes) -> int: + """Compute the 64‐bit FNV‑1 hash of data.""" + h = FNV64_OFFSET_BASIS + for b in data: + h = (h * FNV64_PRIME) & MASK64 + h = h ^ b + return h + + +class SimhashFeature: + def __init__(self, hash_value: int, weight: int = 1): + self.hash_value = hash_value + self.weight = weight + + def sum(self) -> int: + """Return the 64-bit hash (sum) of this feature.""" + return self.hash_value + + def get_weight(self) -> int: + """Return the weight of this feature.""" + return self.weight + + +def new_feature(f: bytes) -> SimhashFeature: + """Return a new feature for the given byte slice with weight 1.""" + return SimhashFeature(fnv1_64(f), 1) + + +def new_feature_with_weight(f: bytes, weight: int) -> SimhashFeature: + """Return a new feature for the given byte slice with the given weight.""" + return SimhashFeature(fnv1_64(f), weight) + + +def vectorize(features: list) -> list: + """ + Given a list of features, return a 64-element vector. + Each feature contributes its weight to each coordinate, + added if that bit is set and subtracted otherwise. + """ + v = [0] * 64 + for feature in features: + h = feature.sum() + w = feature.get_weight() + for i in range(64): + if ((h >> i) & 1) == 1: + v[i] += w + else: + v[i] -= w + return v + + +def vectorize_bytes(features: list) -> list: + """ + Given a list of byte slices, treat each as a feature (with weight 1) + by computing its FNV-1 hash. + """ + v = [0] * 64 + for feat in features: + h = fnv1_64(feat) + for i in range(64): + if ((h >> i) & 1) == 1: + v[i] += 1 + else: + v[i] -= 1 + return v + + +def fingerprint(v: list) -> int: + """ + Given a 64-element vector, return a 64-bit fingerprint. + For each bit i, if v[i] >= 0, set bit i to 1; otherwise leave it 0. + """ + f = 0 + for i in range(64): + if v[i] >= 0: + f |= 1 << i + return f + + +def compare(a: int, b: int) -> int: + """ + Calculate the Hamming distance between two 64-bit integers. + (The number of differing bits.) + """ + v = a ^ b + c = 0 + while v: + v &= v - 1 + c += 1 + return c + + +def simhash(fs) -> int: + """ + Given a feature set (an object with a get_features() method), + return its 64-bit simhash. + """ + return fingerprint(vectorize(fs.get_features())) + + +def simhash_bytes(b: list) -> int: + """ + Given a list of byte slices, return the simhash. + """ + return fingerprint(vectorize_bytes(b)) + + +boundaries = re.compile(rb"[\w']+(?:\://[\w\./]+){0,1}") +unicode_boundaries = re.compile(r"[\w'-]+", re.UNICODE) + + +# --- Helper Functions for Feature Extraction --- +def _get_features_bytes(b: bytes, pattern: re.Pattern) -> list: + """ + Split the given byte string using the given regex pattern, + and return a list of features (each created with new_feature). + """ + words = pattern.findall(b) + return [new_feature(word) for word in words] + + +def _get_features_str(s: str, pattern) -> list: + """ + Split the given string using the given regex pattern, + and return a list of features (each created by encoding to UTF-8). + """ + words = pattern.findall(s) + return [new_feature(word.encode('utf-8')) for word in words] + + +class WordFeatureSet: + def __init__(self, b: bytes): + # Normalize the input to lowercase. + self.b = b.lower() + + def get_features(self) -> list: + return _get_features_bytes(self.b, boundaries) + + +class UnicodeWordFeatureSet: + def __init__(self, b: bytes, norm_form: str = 'NFC'): + # Decode, normalize (using the provided form), and lowercase. + text = b.decode('utf-8') + normalized = unicodedata.normalize(norm_form, text) + self.text = normalized.lower() + + def get_features(self) -> list: + return _get_features_str(self.text, unicode_boundaries) + + +def shingle(w: int, b: list) -> list: + """ + Return the w-shingling of the given set of byte slices. + For example, if b is [b"this", b"is", b"a", b"test"] + and w == 2, the result is [b"this is", b"is a", b"a test"]. + """ + if w < 1: + raise ValueError('simhash.shingle(): k must be a positive integer') + if w == 1: + return b + w = min(w, len(b)) + count = len(b) - w + 1 + shingles = [] + for i in range(count): + shingles.append(b' '.join(b[i : i + w])) + return shingles diff --git a/version.py b/version.py index 022888ab..1ee8cfbf 100755 --- a/version.py +++ b/version.py @@ -23,8 +23,8 @@ THE SOFTWARE. """ -import os import codecs +import os def read(rel_path): @@ -48,8 +48,7 @@ def get_version(rel_path): if line.startswith('__version__'): delim = '"' if '"' in line else "'" return line.split(delim)[1] - else: - raise RuntimeError('Unable to find version string.') + raise RuntimeError('Unable to find version string.') """ From 4edc219fd2658ba497fd596451f98aabf46bdb2d Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 27 Mar 2025 12:29:53 +0100 Subject: [PATCH 297/489] feat: fix lint issues and tests --- src/scanoss/cli.py | 13 ++++--------- src/scanoss/scanossgrpc.py | 22 +++++++++------------- tests/test_file_filters.py | 9 ++++----- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index e6a7bf6d..9a1f7c5a 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -66,11 +66,6 @@ from .threadeddependencies import SCOPE from .utils.file import validate_json_file -DEFAULT_POST_SIZE = 32 -DEFAULT_TIMEOUT = 180 -MIN_TIMEOUT_VALUE = 5 -DEFAULT_RETRY = 5 -PYTHON3_OR_LATER = 3 HEADER_PARTS_COUNT = 2 @@ -589,9 +584,9 @@ def setup_args() -> None: # noqa: PLR0915 # Global Scan/Fingerprint filter options for p in [p_scan, p_wfp]: p.add_argument('--obfuscate', action='store_true', help='Obfuscate fingerprints') - p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions') - p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders') - p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders') + p.add_argument('--all-extensions', action='store_true', help='Fingerprint all file extensions/types...') + p.add_argument('--all-folders', action='store_true', help='Fingerprint all folders...') + p.add_argument('--all-hidden', action='store_true', help='Fingerprint all hidden files/folders...') p.add_argument('--hpsm', '-H', action='store_true', help='Use High Precision Snippet Matching algorithm.') p.add_argument('--skip-snippets', '-S', action='store_true', help='Skip the generation of snippets') p.add_argument('--skip-extension', '-E', type=str, action='append', help='File Extension to skip.') @@ -1282,7 +1277,7 @@ def get_pac_file(pac: str): if pac == 'auto': pac_file = pypac.get_pac() # try to determine the PAC file elif pac.startswith('file://'): - pac_local = pac.strip('file://') + pac_local = pac[7:] # Remove 'file://' prefix (7 characters) if not os.path.exists(pac_local): print_stderr(f'Error: PAC file does not exist: {pac_local}.') sys.exit(1) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 6fb5569d..9271d7eb 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -57,10 +57,7 @@ from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub -from .api.dependencies.v2.scanoss_dependencies_pb2 import ( - DependencyRequest, - DependencyResponse, -) +from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest @@ -77,6 +74,7 @@ MAX_CONCURRENT_REQUESTS = 5 + class ScanossGrpcError(Exception): """ Custom exception for SCANOSS gRPC errors @@ -139,7 +137,6 @@ def __init__( # noqa: PLR0913, PLR0915 self.req_headers = req_headers self.metadata = [] - if self.api_key: self.metadata.append(('x-api-key', api_key)) # Set API key if we have one if ver_details: @@ -528,9 +525,6 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st :return: True if successful, False otherwise """ - SUCCEDED_WITH_WARNINGS_STATUS_CODE = 2 - FAILED_STATUS_CODE = 3 - if not status_response: self.print_stderr(f'Warning: No status response supplied (rqId: {request_id}). Assuming it was ok.') return True @@ -601,18 +595,20 @@ def get_provenance_json(self, purls: dict) -> dict: def load_generic_headers(self): """ - Adds custom headers from req_headers to metadata. + Adds custom headers from req_headers to metadata. - If x-api-key is present and no URL is configured (directly or via - environment), sets URL to the premium endpoint (DEFAULT_URL2). - """ + If x-api-key is present and no URL is configured (directly or via + environment), sets URL to the premium endpoint (DEFAULT_URL2). + """ if self.req_headers: # Load generic headers for key, value in self.req_headers.items(): - if key == 'x-api-key': # Set premium URL if x-api-key header is set + if key == 'x-api-key': # Set premium URL if x-api-key header is set if not self.url and not os.environ.get('SCANOSS_GRPC_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.api_key = value self.metadata.append((key, value)) + + # # End of ScanossGrpc Class # diff --git a/tests/test_file_filters.py b/tests/test_file_filters.py index 8e5dcf3c..eb34face 100644 --- a/tests/test_file_filters.py +++ b/tests/test_file_filters.py @@ -197,16 +197,15 @@ def test_get_filtered_files_from_files(self): os.path.join(self.test_dir, 'file1.js'), os.path.join(self.test_dir, 'file2.css'), # Should be skipped os.path.join(self.test_dir, 'dir1/file3.py'), - os.path.join(self.test_dir, 'dir1/__pycache__/file4.py'), + os.path.join(self.test_dir, 'dir1/__pycache__/file4.py'), # Should be skipped ] self.create_files(files) - filtered_files = self.file_filters.get_filtered_files_from_files(files) + filtered_files = self.file_filters.get_filtered_files_from_files(files, self.test_dir) expected_files = [ - os.path.relpath(os.path.join(self.test_dir, 'file1.js'), os.getcwd()), - os.path.relpath(os.path.join(self.test_dir, 'dir1', 'file3.py'), os.getcwd()), - os.path.relpath(os.path.join(self.test_dir, 'dir1', '__pycache__', 'file4.py'), os.getcwd()), + 'file1.js', + 'dir1/file3.py', ] self.assertEqual(sorted(filtered_files), sorted(expected_files)) From ce782b3ccfe14255590f4f8212dedbbfeabff636 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 27 Mar 2025 14:12:14 +0100 Subject: [PATCH 298/489] feat: fix MINIMUM_FILE_NAME_LENGTH --- src/scanoss/scanners/folder_hasher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 6d4e4af7..902f69b7 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -140,7 +140,7 @@ def _build_root_node(self, path: str) -> DirectoryNode: root_node = DirectoryNode(str(root)) all_files = [ - f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) < MINIMUM_FILE_NAME_LENGTH + f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) <= MINIMUM_FILE_NAME_LENGTH ] filtered_files = self.file_filters.get_filtered_files_from_files(all_files, str(root)) From 81575755f414973dee9c3f3c787b12127644ac5c Mon Sep 17 00:00:00 2001 From: githole Date: Tue, 1 Apr 2025 21:21:24 +0900 Subject: [PATCH 299/489] fix: resolve #106 by avoiding shared mutable queues in ThreadedScanning Move input/output queues into __init__ to ensure instance-level isolation. Did not use `field(default_factory=...)` because __init__ is explicitly defined, which disables dataclass auto-init behavior. --- src/scanoss/threadedscanning.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index da1a180e..d58e6d56 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -49,8 +49,6 @@ class ThreadedScanning(ScanossBase): Multiple threads pull messages off this queue, process the request and put the results into an output queue """ - inputs: queue.Queue = queue.Queue() - output: queue.Queue = queue.Queue() bar: Bar = None def __init__( @@ -65,6 +63,8 @@ def __init__( :param nb_threads: Number of thread to run (default 5) """ super().__init__(debug, trace, quiet) + self.inputs = queue.Queue() + self.output = queue.Queue() self.scanapi = scanapi self.nb_threads = nb_threads self._isatty = sys.stderr.isatty() From a9cd10fc2ecf6ff478d52624b71ebc07996e6230 Mon Sep 17 00:00:00 2001 From: githole Date: Thu, 3 Apr 2025 20:41:55 +0900 Subject: [PATCH 300/489] chore: fix Ruff lint warnings --- src/scanoss/threadedscanning.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index d58e6d56..e9784e3a 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -23,13 +23,13 @@ """ import os +import queue import sys import threading -import queue import time - -from typing import Dict, List from dataclasses import dataclass +from typing import Dict, List + from progress.bar import Bar from .scanossapi import ScanossApi @@ -134,7 +134,7 @@ def queue_add(self, wfp: str) -> None: :param wfp: WFP to add to queue """ if wfp is None or wfp == '': - self.print_stderr(f'Warning: empty WFP. Skipping from scan...') + self.print_stderr('Warning: empty WFP. Skipping from scan...') else: self.inputs.put(wfp) From f3477475f1511b6d99a3349013d5e5e995708987 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Wed, 9 Apr 2025 11:06:47 +0100 Subject: [PATCH 301/489] update hfh proto api defs --- .../api/scanning/v2/scanoss_scanning_pb2.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index af611a33..b81bf7e9 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -16,7 +16,7 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xff\x01\n\nHFHRequest\x12\x12\n\nbest_match\x18\x01 \x01(\x08\x12\x11\n\tthreshold\x18\x02 \x01(\x05\x12:\n\x04root\x18\x03 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x1a\x8d\x01\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\"\xa2\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a?\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x12\n\nconfidence\x18\x03 \x01(\x02\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xff\x01\n\nHFHRequest\x12\x12\n\nbest_match\x18\x01 \x01(\x08\x12\x11\n\tthreshold\x18\x02 \x01(\x05\x12:\n\x04root\x18\x03 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x1a\x8d\x01\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\"\xc1\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x1a\x81\x01\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component\x12\x13\n\x0bprobability\x18\x03 \x01(\x02\x12\r\n\x05stage\x18\x04 \x01(\x05\x32\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) @@ -33,11 +33,11 @@ _HFHREQUEST_CHILDREN._serialized_start=310 _HFHREQUEST_CHILDREN._serialized_end=451 _HFHRESPONSE._serialized_start=454 - _HFHRESPONSE._serialized_end=744 + _HFHRESPONSE._serialized_end=775 _HFHRESPONSE_COMPONENT._serialized_start=586 - _HFHRESPONSE_COMPONENT._serialized_end=649 - _HFHRESPONSE_RESULT._serialized_start=651 - _HFHRESPONSE_RESULT._serialized_end=744 - _SCANNING._serialized_start=747 - _SCANNING._serialized_end=1004 + _HFHRESPONSE_COMPONENT._serialized_end=643 + _HFHRESPONSE_RESULT._serialized_start=646 + _HFHRESPONSE_RESULT._serialized_end=775 + _SCANNING._serialized_start=778 + _SCANNING._serialized_end=1035 # @@protoc_insertion_point(module_scope) From 09df227cc0214795907bb7d25892834257d64574 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Wed, 9 Apr 2025 18:19:49 +0100 Subject: [PATCH 302/489] add transitive dependency grpc api --- .../api/common/v2/scanoss_common_pb2.py | 8 +++-- .../v2/scanoss_dependencies_pb2.py | 14 ++++++-- .../v2/scanoss_dependencies_pb2_grpc.py | 34 +++++++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index 23546c71..cbcd2e5b 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\")\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) @@ -21,8 +21,8 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _STATUSCODE._serialized_start=336 - _STATUSCODE._serialized_end=432 + _STATUSCODE._serialized_start=379 + _STATUSCODE._serialized_end=475 _STATUSRESPONSE._serialized_start=69 _STATUSRESPONSE._serialized_end=153 _ECHOREQUEST._serialized_start=155 @@ -33,4 +33,6 @@ _PURLREQUEST._serialized_end=334 _PURLREQUEST_PURLS._serialized_start=292 _PURLREQUEST_PURLS._serialized_end=334 + _PURL._serialized_start=336 + _PURL._serialized_end=377 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index 0090b4cd..a0779b29 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -16,7 +16,7 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies2\xa8\x02\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies\"z\n\x1bTransitiveDependencyRequest\x12\x11\n\tecosystem\x18\x01 \x01(\t\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x12\r\n\x05limit\x18\x03 \x01(\x05\x12*\n\x05purls\x18\x05 \x03(\x0b\x32\x1b.scanoss.api.common.v2.Purl\"\xe2\x01\n\x1cTransitiveDependencyResponse\x12\\\n\x0c\x64\x65pendencies\x18\x01 \x03(\x0b\x32\x46.scanoss.api.dependencies.v2.TransitiveDependencyResponse.Dependencies\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a-\n\x0c\x44\x65pendencies\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t2\xe7\x03\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*\x12\xbc\x01\n\x19GetTransitiveDependencies\x12\x38.scanoss.api.dependencies.v2.TransitiveDependencyRequest\x1a\x39.scanoss.api.dependencies.v2.TransitiveDependencyResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/dependencies/transitive:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) @@ -28,6 +28,8 @@ _DEPENDENCIES.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' _DEPENDENCIES.methods_by_name['GetDependencies']._options = None _DEPENDENCIES.methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' + _DEPENDENCIES.methods_by_name['GetTransitiveDependencies']._options = None + _DEPENDENCIES.methods_by_name['GetTransitiveDependencies']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/dependencies/transitive:\001*' _DEPENDENCYREQUEST._serialized_start=208 _DEPENDENCYREQUEST._serialized_end=447 _DEPENDENCYREQUEST_PURLS._serialized_start=313 @@ -42,6 +44,12 @@ _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=850 _DEPENDENCYRESPONSE_FILES._serialized_start=853 _DEPENDENCYRESPONSE_FILES._serialized_end=986 - _DEPENDENCIES._serialized_start=989 - _DEPENDENCIES._serialized_end=1285 + _TRANSITIVEDEPENDENCYREQUEST._serialized_start=988 + _TRANSITIVEDEPENDENCYREQUEST._serialized_end=1110 + _TRANSITIVEDEPENDENCYRESPONSE._serialized_start=1113 + _TRANSITIVEDEPENDENCYRESPONSE._serialized_end=1339 + _TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=1294 + _TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=1339 + _DEPENDENCIES._serialized_start=1342 + _DEPENDENCIES._serialized_end=1829 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py index cddf7cfa..ae675b67 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py @@ -27,6 +27,11 @@ def __init__(self, channel): request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, ) + self.GetTransitiveDependencies = channel.unary_unary( + '/scanoss.api.dependencies.v2.Dependencies/GetTransitiveDependencies', + request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyResponse.FromString, + ) class DependenciesServicer(object): @@ -48,6 +53,13 @@ def GetDependencies(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def GetTransitiveDependencies(self, request, context): + """Get transitive dependency details + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_DependenciesServicer_to_server(servicer, server): rpc_method_handlers = { @@ -61,6 +73,11 @@ def add_DependenciesServicer_to_server(servicer, server): request_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.FromString, response_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.SerializeToString, ), + 'GetTransitiveDependencies': grpc.unary_unary_rpc_method_handler( + servicer.GetTransitiveDependencies, + request_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyRequest.FromString, + response_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.dependencies.v2.Dependencies', rpc_method_handlers) @@ -106,3 +123,20 @@ def GetDependencies(request, scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetTransitiveDependencies(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/GetTransitiveDependencies', + scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyRequest.SerializeToString, + scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From 9967590b406efecc845a71f8bf4e7f3c27cae580 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 26 Mar 2025 10:19:33 +0100 Subject: [PATCH 303/489] feat: ES-213 Initialize container scanning with syft --- Dockerfile | 15 +- src/scanoss/cli.py | 57 +++++- src/scanoss/scanners/container_scanner.py | 233 ++++++++++++++++++++++ 3 files changed, 292 insertions(+), 13 deletions(-) create mode 100644 src/scanoss/scanners/container_scanner.py diff --git a/Dockerfile b/Dockerfile index fc066e50..063fd113 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,9 @@ FROM base AS builder # Setup the required build tooling RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential gcc \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + && apt-get install -y --no-install-recommends build-essential gcc \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Create and activate virtual environment RUN python -m venv /opt/venv @@ -56,9 +56,12 @@ ENV GRPC_POLL_STRATEGY=poll # Install jq and curl commands RUN apt-get update \ - && apt-get install -y --no-install-recommends jq curl \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + && apt-get install -y --no-install-recommends jq curl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install syft +RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin # Setup working directory and user WORKDIR /scanoss diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 9a1f7c5a..8ac4e276 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -31,6 +31,12 @@ import pypac +from scanoss.scanners.container_scanner import ( + DEFAULT_SYFT_COMMAND, + DEFAULT_SYFT_TIMEOUT, + ContainerScanner, + create_container_scanner_config_from_args, +) from scanoss.scanners.folder_hasher import ( FolderHasher, create_folder_hasher_config_from_args, @@ -197,8 +203,7 @@ def setup_args() -> None: # noqa: PLR0915 description=f'Produce dependency file summary: {__version__}', help='Scan source code for dependencies, but do not decorate them', ) - p_dep.set_defaults(func=dependency) - p_dep.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') + p_dep.add_argument('scan_loc', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') p_dep.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_dep.add_argument( '--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' @@ -209,6 +214,27 @@ def setup_args() -> None: # noqa: PLR0915 default=600, help='Timeout (in seconds) for scancode to complete (optional - default 600)', ) + p_dep.add_argument( + '--container', + type=str, + help=( + 'Container image to scan. Supports yourrepo/yourimage:tag, Docker tar, ' + 'OCI tar, OCI directory, SIF Container, or generic filesystem directory.' + ), + ) + p_dep.add_argument( + '--syft-command', + type=str, + help='Syft command and path if required (optional - default syft).', + default=DEFAULT_SYFT_COMMAND, + ) + p_dep.add_argument( + '--syft-timeout', + type=int, + default=DEFAULT_SYFT_TIMEOUT, + help='Timeout (in seconds) for syft to complete (optional - default 600)', + ) + p_dep.set_defaults(func=dependency) # Sub-command: file_count p_fc = subparsers.add_parser( @@ -1052,12 +1078,29 @@ def dependency(parser, args): args: Namespace Parsed arguments """ - if not args.scan_dir: - print_stderr('Please specify a file/folder') + if not args.scan_loc and not args.container: + print_stderr('Please specify a file/folder or container') parser.parse_args([args.subparser, '-h']) sys.exit(1) - if not os.path.exists(args.scan_dir): - print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') + + # Container scanning + + if args.container: + try: + container_scanner_config = create_container_scanner_config_from_args(args) + container_scanner = ContainerScanner(config=container_scanner_config, what_to_scan=args.container) + + container_scanner.scan() + return container_scanner.present( + output_file=args.output, output_format='json' + ) # TODO: support other formats? + except Exception as e: + print_stderr(f'ERROR: {e}') + sys.exit(1) + + # File/folder scanning + if not os.path.exists(args.scan_loc): + print_stderr(f'Error: File or folder specified does not exist: {args.scan_loc}.') sys.exit(1) scan_output: str = None if args.output: @@ -1067,7 +1110,7 @@ def dependency(parser, args): sc_deps = ScancodeDeps( debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout ) - if not sc_deps.get_dependencies(what_to_scan=args.scan_dir, result_output=scan_output): + if not sc_deps.get_dependencies(what_to_scan=args.scan_loc, result_output=scan_output): sys.exit(1) diff --git a/src/scanoss/scanners/container_scanner.py b/src/scanoss/scanners/container_scanner.py new file mode 100644 index 00000000..7fdcce92 --- /dev/null +++ b/src/scanoss/scanners/container_scanner.py @@ -0,0 +1,233 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import os +import subprocess +from dataclasses import dataclass +from typing import List, Optional, TypedDict + +from scanoss.scanossbase import ScanossBase +from scanoss.utils.abstract_presenter import AbstractPresenter + +DEFAULT_SYFT_TIMEOUT = 600 +DEFAULT_SYFT_COMMAND = 'syft' + + +@dataclass +class ContainerScannerConfig: + debug: bool = False + trace: bool = False + quiet: bool = False + syft_command: str = DEFAULT_SYFT_COMMAND + syft_timeout: int = DEFAULT_SYFT_TIMEOUT + + +def create_container_scanner_config_from_args(args) -> ContainerScannerConfig: + return ContainerScannerConfig( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + syft_command=args.syft_command, + syft_timeout=args.syft_timeout, + ) + + +class LicenseItem(TypedDict): + value: str + spdxExpression: str + type: str + + +class SyftArtifactItem(TypedDict): + name: str + version: str + type: str + purl: str + licenses: List[LicenseItem] + + +class SyftScanResult(TypedDict): + artifacts: List[SyftArtifactItem] + + +class SyftScanError(Exception): + """Base exception for Syft scan errors""" + + pass + + +class SyftExecutionError(SyftScanError): + """Raised when Syft returns a non-zero exit code""" + + pass + + +class SyftJsonError(SyftScanError): + """Raised when Syft output cannot be parsed as JSON""" + + pass + + +class SyftTimeoutError(SyftScanError): + """Raised when a Syft scan times out""" + + pass + + +class ContainerScanner: + """SCANOSS container scanning class. + + This class provides functionality to scan containers using Syft and process + the results into SCANOSS dependency format. + """ + + def __init__( + self, + config: ContainerScannerConfig, + what_to_scan: str, + ): + """Initialize ContainerScanner class. + + Args: + config: ContainerScannerConfig object containing configuration settings. + """ + self.base = ScanossBase( + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + self.presenter = ContainerScannerPresenter( + self, + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + self.what_to_scan: str = what_to_scan + self.syft_command: str = config.syft_command + self.syft_timeout: int = config.syft_timeout + self.scan_results: Optional[SyftScanResult] = None + + def scan(self) -> SyftScanResult: + """ + Scan the provided container using Syft. + + Returns: + SyftScanResult: The Syft scan results. + + Raises: + SyftExecutionError: If Syft returns a non-zero exit code + SyftJsonError: If the scan output cannot be parsed as JSON + SyftTimeoutError: If the scan times out + SyftScanError: For other scan-related errors + """ + self.run_scan() + return self.scan_results + + def run_scan( + self, + ) -> None: + """Run a syft scan of the specified target. + + Raises: + SyftExecutionError: If Syft returns a non-zero exit code + SyftJsonError: If the scan output cannot be parsed as JSON + SyftTimeoutError: If the scan times out + SyftScanError: For other scan-related errors + """ + try: + self.base.print_trace( + f'About to execute {self.syft_command} scan {self.what_to_scan} -q {self.what_to_scan} -o json' + ) + result = subprocess.run( + [self.syft_command, 'scan', self.what_to_scan, '-o', 'json'], + cwd=os.getcwd(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=self.syft_timeout, + check=False, + ) + self.base.print_trace(f'Subprocess return: {result}') + + if result.returncode: + error_msg = ( + f'Syft scan of {self.what_to_scan} failed with exit code {result.returncode}:\n{result.stdout}' + ) + self.base.print_stderr(f'ERROR: {error_msg}') + raise SyftExecutionError(error_msg) + + try: + json_data = json.loads(result.stdout) + self.scan_results = json_data + except json.JSONDecodeError as e: + error_msg = f'Failed to parse JSON output from syft: {e}\n{result.stdout}' + self.base.print_stderr(f'ERROR: {error_msg}') + raise SyftJsonError(error_msg) from e + + except subprocess.TimeoutExpired as e: + error_msg = f'Timed out attempting to run syft scan on {self.what_to_scan}: {e}' + self.base.print_stderr(f'ERROR: {error_msg}') + raise SyftTimeoutError(error_msg) from e + + except Exception as e: + if isinstance(e, SyftScanError): + raise + error_msg = f'Issue running syft scan on {self.what_to_scan}: {e}' + self.base.print_stderr(f'ERROR: {error_msg}') + raise SyftScanError(error_msg) from e + + def present(self, output_format: str = None, output_file: str = None): + """Present the results in the selected format""" + self.presenter.present(output_format=output_format, output_file=output_file) + + +class ContainerScannerPresenter(AbstractPresenter): + """ + ContainerScannerPresenter presenter class + Handles the presentation of the container scan results + """ + + def __init__(self, scanner: ContainerScanner, **kwargs): + super().__init__(**kwargs) + self.scanner = scanner + + def _format_json_output(self) -> str: + """ + Format the scan output data into a JSON object + + Returns: + str: The formatted JSON string + """ + return json.dumps(self.scanner.scan_results, indent=2) + + def _format_plain_output(self) -> str: + """ + Format the scan output data into a plain text string + """ + return ( + json.dumps(self.scanner.scan_results, indent=2) + if isinstance(self.scanner.scan_results, dict) + else str(self.scanner.scan_results) + ) From bdb065a42a63c3f10ac03fbbcc4212d89d470984 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 27 Mar 2025 10:30:11 +0100 Subject: [PATCH 304/489] feat: ES-213 Normalize syft output into our format --- src/scanoss/cli.py | 1 - src/scanoss/scanners/container_scanner.py | 64 +++++++++++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 8ac4e276..3b054da0 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1084,7 +1084,6 @@ def dependency(parser, args): sys.exit(1) # Container scanning - if args.container: try: container_scanner_config = create_container_scanner_config_from_args(args) diff --git a/src/scanoss/scanners/container_scanner.py b/src/scanoss/scanners/container_scanner.py index 7fdcce92..5fdffcc7 100644 --- a/src/scanoss/scanners/container_scanner.py +++ b/src/scanoss/scanners/container_scanner.py @@ -59,6 +59,10 @@ class LicenseItem(TypedDict): spdxExpression: str type: str + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + class SyftArtifactItem(TypedDict): name: str @@ -67,10 +71,24 @@ class SyftArtifactItem(TypedDict): purl: str licenses: List[LicenseItem] + @classmethod + def from_dict(cls, data: dict): + return cls( + name=data['name'], + version=data['version'], + type=data['type'], + purl=data['purl'], + licenses=[LicenseItem.from_dict(lic) for lic in data['licenses']], + ) + class SyftScanResult(TypedDict): artifacts: List[SyftArtifactItem] + @classmethod + def from_dict(cls, data: dict): + return cls(artifacts=[SyftArtifactItem.from_dict(a) for a in data['artifacts']]) + class SyftScanError(Exception): """Base exception for Syft scan errors""" @@ -96,6 +114,20 @@ class SyftTimeoutError(SyftScanError): pass +class PurlItem(TypedDict): + purl: str + requirement: Optional[str] + + +class ContainerScanResultFileItem(TypedDict): + file: str + purls: List[PurlItem] + + +class ContainerScanResult(TypedDict): + files: List[ContainerScanResultFileItem] + + class ContainerScanner: """SCANOSS container scanning class. @@ -161,7 +193,7 @@ def run_scan( f'About to execute {self.syft_command} scan {self.what_to_scan} -q {self.what_to_scan} -o json' ) result = subprocess.run( - [self.syft_command, 'scan', self.what_to_scan, '-o', 'json'], + [self.syft_command, 'scan', self.what_to_scan, '-q', '-o', 'json'], cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -180,7 +212,7 @@ def run_scan( try: json_data = json.loads(result.stdout) - self.scan_results = json_data + self.scan_results = SyftScanResult.from_dict(json_data) except json.JSONDecodeError as e: error_msg = f'Failed to parse JSON output from syft: {e}\n{result.stdout}' self.base.print_stderr(f'ERROR: {error_msg}') @@ -220,14 +252,36 @@ def _format_json_output(self) -> str: Returns: str: The formatted JSON string """ - return json.dumps(self.scanner.scan_results, indent=2) + return json.dumps(self._normalize_syft_output(), indent=2) + + def _normalize_syft_output(self) -> ContainerScanResult: + """ + Normalize the Syft output data into the same format we use in dependency scanning + + Returns: + ContainerScanResult: The normalized output + """ + normalized_output = ContainerScanResult() + + # This is a workaround because we don't have file paths as in dependency scanning, we use the container name + file_name = self.scanner.what_to_scan + artifacts = self.scanner.scan_results['artifacts'] + + normalized_output['files'] = [ + { + 'file': file_name, + 'purls': [PurlItem(purl=artifact['purl']) for artifact in artifacts], + } + ] + + return normalized_output def _format_plain_output(self) -> str: """ Format the scan output data into a plain text string """ return ( - json.dumps(self.scanner.scan_results, indent=2) - if isinstance(self.scanner.scan_results, dict) + json.dumps(self._normalize_syft_output(), indent=2) + if isinstance(self.scanner.scan_results, SyftScanResult) else str(self.scanner.scan_results) ) From 69ad4e0c8ffbe0b71b0fb44baba01c462050ab9e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 23 Apr 2025 11:04:24 +0200 Subject: [PATCH 305/489] feat: ES-213 Create separate container-scan command feat: ES-213 Decorate container scan results with dependencies feat: ES-213 Support spdxlite, cyclonedx csv outputs feat: ES-213 Small fix --- src/scanoss/cli.py | 133 +++++++---- src/scanoss/results.py | 9 + src/scanoss/scanners/container_scanner.py | 269 +++++++++++++++++----- src/scanoss/scanners/folder_hasher.py | 9 + src/scanoss/scanners/scanner_hfh.py | 9 + src/scanoss/scanossbase.py | 6 + src/scanoss/utils/abstract_presenter.py | 34 ++- 7 files changed, 370 insertions(+), 99 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 3b054da0..f61eb880 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -117,14 +117,6 @@ def setup_args() -> None: # noqa: PLR0915 p_scan.add_argument('--files', '-e', type=str, nargs='*', help='List of files to scan.') p_scan.add_argument('--identify', '-i', type=str, help='Scan and identify components in SBOM file') p_scan.add_argument('--ignore', '-n', type=str, help='Ignore components specified in the SBOM file') - p_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') - p_scan.add_argument( - '--format', - '-f', - type=str, - choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], - help='Result output format (optional - default: plain)', - ) p_scan.add_argument( '--threads', '-T', type=int, default=5, help='Number of threads to use while scanning (optional - default 5)' ) @@ -194,7 +186,6 @@ def setup_args() -> None: # noqa: PLR0915 type=str, help='Fingerprint the file contents supplied via STDIN (optional)', ) - p_wfp.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') # Sub-command: dependency p_dep = subparsers.add_parser( @@ -204,7 +195,6 @@ def setup_args() -> None: # noqa: PLR0915 help='Scan source code for dependencies, but do not decorate them', ) p_dep.add_argument('scan_loc', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') - p_dep.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_dep.add_argument( '--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' ) @@ -214,27 +204,48 @@ def setup_args() -> None: # noqa: PLR0915 default=600, help='Timeout (in seconds) for scancode to complete (optional - default 600)', ) - p_dep.add_argument( - '--container', + p_dep.set_defaults(func=dependency) + + # Container scan sub-command + p_cs = subparsers.add_parser( + 'container-scan', + aliases=['cs'], + description=f'Analyse/scan the given container image: {__version__}', + help='Scan container image', + ) + p_cs.add_argument( + 'scan_loc', + metavar='IMAGE', type=str, + nargs='?', help=( 'Container image to scan. Supports yourrepo/yourimage:tag, Docker tar, ' 'OCI tar, OCI directory, SIF Container, or generic filesystem directory.' ), ) - p_dep.add_argument( + p_cs.add_argument( + '--retry', '-R', type=int, default=5, help='Retry limit for API communication (optional - default 5)' + ) + p_cs.add_argument( + '--timeout', + '-M', + type=int, + default=180, + help='Timeout (in seconds) for API communication (optional - default 180)', + ) + p_cs.add_argument( '--syft-command', type=str, help='Syft command and path if required (optional - default syft).', default=DEFAULT_SYFT_COMMAND, ) - p_dep.add_argument( + p_cs.add_argument( '--syft-timeout', type=int, default=DEFAULT_SYFT_TIMEOUT, help='Timeout (in seconds) for syft to complete (optional - default 600)', ) - p_dep.set_defaults(func=dependency) + p_cs.set_defaults(func=container_scan) # Sub-command: file_count p_fc = subparsers.add_parser( @@ -245,7 +256,6 @@ def setup_args() -> None: # noqa: PLR0915 ) p_fc.set_defaults(func=file_count) p_fc.add_argument('scan_dir', metavar='DIR', type=str, nargs='?', help='A folder to search') - p_fc.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_fc.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders') # Sub-command: convert @@ -257,7 +267,6 @@ def setup_args() -> None: # noqa: PLR0915 ) p_cnv.set_defaults(func=convert) p_cnv.add_argument('--input', '-i', type=str, required=True, help='Input file name') - p_cnv.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_cnv.add_argument( '--format', '-f', @@ -356,7 +365,6 @@ def setup_args() -> None: # noqa: PLR0915 # Common Component sub-command options for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: - p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p.add_argument( '--timeout', '-M', @@ -404,7 +412,6 @@ def setup_args() -> None: # noqa: PLR0915 p_c_dwnld.add_argument( '--port', '-p', required=False, type=int, default=443, help='Server port number (default: 443).' ) - p_c_dwnld.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') # Utils Sub-command: utils pac-proxy p_p_proxy = utils_sub.add_parser( @@ -520,7 +527,6 @@ def setup_args() -> None: # noqa: PLR0915 help='Scan the given directory using folder hashing', ) p_folder_scan.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='The root directory to scan') - p_folder_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_folder_scan.add_argument( '--timeout', '-M', @@ -561,7 +567,6 @@ def setup_args() -> None: # noqa: PLR0915 help='Produce a folder hash for the given directory', ) p_folder_hash.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') - p_folder_hash.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') p_folder_hash.add_argument( '--format', '-f', @@ -572,6 +577,36 @@ def setup_args() -> None: # noqa: PLR0915 ) p_folder_hash.set_defaults(func=folder_hash) + # Output options + for p in [ + p_scan, + p_cs, + p_wfp, + p_dep, + p_fc, + p_cnv, + c_crypto, + c_vulns, + c_search, + c_versions, + c_semgrep, + c_provenance, + p_c_dwnld, + p_folder_scan, + p_folder_hash, + ]: + p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') + + # Format options + for p in [p_scan, p_cs]: + p.add_argument( + '--format', + '-f', + type=str, + choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], + help='Result output format (optional - default: plain)', + ) + # Scanoss settings options for p in [p_folder_scan, p_scan, p_wfp, p_folder_hash]: p.add_argument( @@ -601,7 +636,7 @@ def setup_args() -> None: # noqa: PLR0915 p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file') # Global Scan command options - for p in [p_scan]: + for p in [p_scan, p_cs]: p.add_argument( '--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)' ) @@ -629,7 +664,7 @@ def setup_args() -> None: # noqa: PLR0915 p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.') # Global Scan/GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, p_folder_scan, p_cs]: p.add_argument( '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' ) @@ -655,7 +690,7 @@ def setup_args() -> None: # noqa: PLR0915 ) # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, p_folder_scan, p_cs]: p.add_argument( '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' ) @@ -693,6 +728,8 @@ def setup_args() -> None: # noqa: PLR0915 p_copyleft, c_provenance, p_folder_scan, + p_folder_hash, + p_cs, ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') @@ -1078,26 +1115,11 @@ def dependency(parser, args): args: Namespace Parsed arguments """ - if not args.scan_loc and not args.container: - print_stderr('Please specify a file/folder or container') + if not args.scan_loc: + print_stderr('Please specify a file/folder') parser.parse_args([args.subparser, '-h']) sys.exit(1) - # Container scanning - if args.container: - try: - container_scanner_config = create_container_scanner_config_from_args(args) - container_scanner = ContainerScanner(config=container_scanner_config, what_to_scan=args.container) - - container_scanner.scan() - return container_scanner.present( - output_file=args.output, output_format='json' - ) # TODO: support other formats? - except Exception as e: - print_stderr(f'ERROR: {e}') - sys.exit(1) - - # File/folder scanning if not os.path.exists(args.scan_loc): print_stderr(f'Error: File or folder specified does not exist: {args.scan_loc}.') sys.exit(1) @@ -1696,6 +1718,35 @@ def folder_hash(parser, args): sys.exit(1) +def container_scan(parser, args): + """ + Run the "container-scan" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if not args.scan_loc: + print_stderr( + 'Please specify a container image, Docker tar, OCI tar, OCI directory, SIF Container, or directory to scan' + ) + parser.parse_args([args.subparser, '-h']) + sys.exit(1) + + try: + container_scanner_config = create_container_scanner_config_from_args(args) + container_scanner = ContainerScanner(config=container_scanner_config, what_to_scan=args.scan_loc) + + container_scanner.scan() + container_scanner.decorate_scan_results_with_dependencies() + container_scanner.present(output_file=args.output, output_format=args.format) + except Exception as e: + print_stderr(f'ERROR: {e}') + sys.exit(1) + + def get_scanoss_settings_from_args(args): scanoss_settings = None if not args.skip_settings_file: diff --git a/src/scanoss/results.py b/src/scanoss/results.py index 5354b124..b67c980c 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -83,6 +83,15 @@ def _format_json_output(self) -> str: self.base.print_stderr(f'ERROR: Problem formatting JSON output: {e}') return '' + def _format_cyclonedx_output(self) -> str: + pass + + def _format_spdxlite_output(self) -> str: + pass + + def _format_csv_output(self) -> str: + pass + def _format_plain_output(self) -> str: """Format the output data into a plain text string diff --git a/src/scanoss/scanners/container_scanner.py b/src/scanoss/scanners/container_scanner.py index 5fdffcc7..43fd6048 100644 --- a/src/scanoss/scanners/container_scanner.py +++ b/src/scanoss/scanners/container_scanner.py @@ -26,9 +26,14 @@ import os import subprocess from dataclasses import dataclass -from typing import List, Optional, TypedDict +from typing import Dict, List, Optional, TypedDict +from scanoss.constants import DEFAULT_RETRY, DEFAULT_TIMEOUT +from scanoss.csvoutput import CsvOutput +from scanoss.cyclonedx import CycloneDx from scanoss.scanossbase import ScanossBase +from scanoss.scanossgrpc import ScanossGrpc +from scanoss.spdxlite import SpdxLite from scanoss.utils.abstract_presenter import AbstractPresenter DEFAULT_SYFT_TIMEOUT = 600 @@ -40,6 +45,17 @@ class ContainerScannerConfig: debug: bool = False trace: bool = False quiet: bool = False + retry: int = DEFAULT_RETRY + timeout: int = DEFAULT_TIMEOUT + output: Optional[str] = None + format: Optional[str] = None + apiurl: Optional[str] = None + ignore_cert_errors: bool = False + key: Optional[str] = None + proxy: Optional[str] = None + pac: Optional[str] = None + grpc_proxy: Optional[str] = None + ca_cert: Optional[str] = None syft_command: str = DEFAULT_SYFT_COMMAND syft_timeout: int = DEFAULT_SYFT_TIMEOUT @@ -49,6 +65,17 @@ def create_container_scanner_config_from_args(args) -> ContainerScannerConfig: debug=args.debug, trace=args.trace, quiet=args.quiet, + retry=args.retry, + timeout=args.timeout, + output=args.output, + format=args.format, + apiurl=args.api2url, + proxy=args.proxy, + pac=args.pac, + grpc_proxy=args.grpc_proxy, + ca_cert=args.ca_cert, + ignore_cert_errors=args.ignore_cert_errors, + key=args.key, syft_command=args.syft_command, syft_timeout=args.syft_timeout, ) @@ -90,6 +117,43 @@ def from_dict(cls, data: dict): return cls(artifacts=[SyftArtifactItem.from_dict(a) for a in data['artifacts']]) +class PurlItem(TypedDict): + purl: str + requirement: Optional[str] + + +class ContainerScanResultFileItem(TypedDict): + file: str + purls: List[PurlItem] + + +class ContainerScanResult(TypedDict): + files: List[ContainerScanResultFileItem] + + +class DependencyLicenseItem(TypedDict): + value: str + spdxExpression: str + type: str + + +class DependencyItem(TypedDict): + purl: str + licenses: List[DependencyLicenseItem] + + +class DecoratedContainerScanResultFileItem(TypedDict): + file: str + id: str + status: str + dependencies: List[DependencyItem] + + +class DecoratedContainerScanResult(TypedDict): + files: List[ContainerScanResultFileItem] + status: Dict[str, str] + + class SyftScanError(Exception): """Base exception for Syft scan errors""" @@ -114,18 +178,16 @@ class SyftTimeoutError(SyftScanError): pass -class PurlItem(TypedDict): - purl: str - requirement: Optional[str] +class SCANOSSDependencyScanError(Exception): + """Base exception for SCANOSS dependency scan errors""" + pass -class ContainerScanResultFileItem(TypedDict): - file: str - purls: List[PurlItem] +class DecorateScanResultsError(SCANOSSDependencyScanError): + """Raised when there is an issue decorating scan results with dependencies""" -class ContainerScanResult(TypedDict): - files: List[ContainerScanResultFileItem] + pass class ContainerScanner: @@ -155,38 +217,74 @@ def __init__( debug=config.debug, trace=config.trace, quiet=config.quiet, + output_file=config.output, + output_format=config.format, + ) + self.grpc_api = ScanossGrpc( + debug=config.debug, + quiet=config.quiet, + trace=config.trace, + url=config.apiurl, + api_key=config.key, + ca_cert=config.ca_cert, + proxy=config.proxy, + pac=config.pac, + grpc_proxy=config.grpc_proxy, ) self.what_to_scan: str = what_to_scan self.syft_command: str = config.syft_command self.syft_timeout: int = config.syft_timeout - self.scan_results: Optional[SyftScanResult] = None + self.syft_output: Optional[SyftScanResult] = None + self.normalized_syft_output: Optional[ContainerScanResult] = None + self.decorated_scan_results: Optional[DecoratedContainerScanResult] = None - def scan(self) -> SyftScanResult: + def decorate_scan_results_with_dependencies(self) -> None: + """ + Decorate the scan results with dependencies. """ - Scan the provided container using Syft. + try: + decorated_scan_results = self.grpc_api.get_dependencies(self.normalized_syft_output) + self.decorated_scan_results = decorated_scan_results + return decorated_scan_results + except Exception as e: + error_msg = f'Issue decorating scan results with dependencies: {e}' + self.base.print_stderr(f'ERROR: {error_msg}') + raise DecorateScanResultsError(error_msg) from e + + def scan( + self, + ) -> ContainerScanResult: + """Run a syft scan of the specified target. Returns: - SyftScanResult: The Syft scan results. + ContainerScanResult: The container scan results. Raises: - SyftExecutionError: If Syft returns a non-zero exit code - SyftJsonError: If the scan output cannot be parsed as JSON - SyftTimeoutError: If the scan times out SyftScanError: For other scan-related errors """ - self.run_scan() - return self.scan_results + try: + self.syft_output = self._execute_syft_scan() + self.normalized_syft_output = self._normalize_syft_output() + return self.normalized_syft_output + except Exception as e: + if isinstance(e, SyftScanError): + raise + error_msg = f'Issue running syft scan on {self.what_to_scan}: {e}' + self.base.print_stderr(f'ERROR: {error_msg}') + raise SyftScanError(error_msg) from e - def run_scan( - self, - ) -> None: - """Run a syft scan of the specified target. + def _execute_syft_scan(self) -> SyftScanResult: + """ + Execute a Syft scan of the specified target. + + Returns: + SyftScanResult: The result of the Syft scan. Raises: - SyftExecutionError: If Syft returns a non-zero exit code - SyftJsonError: If the scan output cannot be parsed as JSON - SyftTimeoutError: If the scan times out - SyftScanError: For other scan-related errors + SyftScanError: If the Syft scan fails. + SyftJsonError: If the Syft scan output cannot be parsed as JSON. + SyftTimeoutError: If the Syft scan times out. + SyftExecutionError: If the Syft scan execution fails. """ try: self.base.print_trace( @@ -212,7 +310,7 @@ def run_scan( try: json_data = json.loads(result.stdout) - self.scan_results = SyftScanResult.from_dict(json_data) + return SyftScanResult.from_dict(json_data) except json.JSONDecodeError as e: error_msg = f'Failed to parse JSON output from syft: {e}\n{result.stdout}' self.base.print_stderr(f'ERROR: {error_msg}') @@ -230,6 +328,55 @@ def run_scan( self.base.print_stderr(f'ERROR: {error_msg}') raise SyftScanError(error_msg) from e + def _get_dependencies(self) -> None: + """ + Run a dependency scan of the specified target. + """ + try: + if not self.normalized_syft_output: + error_msg = 'Syft scan output is not available' + self.base.print_stderr(error_msg) + raise ValueError(error_msg) + if not self.grpc_api.get_dependencies(self.normalized_syft_output): + error_msg = 'Failed to get dependencies' + self.base.print_stderr(error_msg) + raise SCANOSSDependencyScanError(error_msg) + except Exception as e: + error_msg = f'Failed to run dependency scan: {e}' + self.base.print_stderr(error_msg) + raise SCANOSSDependencyScanError(error_msg) + + def _normalize_syft_output(self) -> ContainerScanResult: + """ + Normalize the Syft output data into the same format we use in dependency scanning + + Returns: + ContainerScanResult: The normalized output + """ + normalized_output = ContainerScanResult() + + # This is a workaround because we don't have file paths as in dependency scanning, we use the container name + file_name = self.what_to_scan + artifacts = self.syft_output['artifacts'] + + unique_purls = set() + unique_purl_items = [] + + for artifact in artifacts: + purl = artifact['purl'] + if purl not in unique_purls: + unique_purls.add(purl) + unique_purl_items.append(PurlItem(purl=purl)) + + normalized_output['files'] = [ + { + 'file': file_name, + 'purls': unique_purl_items, + } + ] + + return normalized_output + def present(self, output_format: str = None, output_file: str = None): """Present the results in the selected format""" self.presenter.present(output_format=output_format, output_file=output_file) @@ -244,6 +391,7 @@ class ContainerScannerPresenter(AbstractPresenter): def __init__(self, scanner: ContainerScanner, **kwargs): super().__init__(**kwargs) self.scanner = scanner + self.AVAILABLE_OUTPUT_FORMATS = ['plain', 'cyclonedx', 'spdxlite', 'csv'] def _format_json_output(self) -> str: """ @@ -252,36 +400,49 @@ def _format_json_output(self) -> str: Returns: str: The formatted JSON string """ - return json.dumps(self._normalize_syft_output(), indent=2) + return json.dumps(self.scanner.decorated_scan_results, indent=2) - def _normalize_syft_output(self) -> ContainerScanResult: + def _format_plain_output(self) -> str: """ - Normalize the Syft output data into the same format we use in dependency scanning - - Returns: - ContainerScanResult: The normalized output + Format the scan output data into a plain text string """ - normalized_output = ContainerScanResult() - - # This is a workaround because we don't have file paths as in dependency scanning, we use the container name - file_name = self.scanner.what_to_scan - artifacts = self.scanner.scan_results['artifacts'] - - normalized_output['files'] = [ - { - 'file': file_name, - 'purls': [PurlItem(purl=artifact['purl']) for artifact in artifacts], - } - ] - - return normalized_output + return json.dumps(self.scanner.decorated_scan_results, indent=2) - def _format_plain_output(self) -> str: + def _format_cyclonedx_output(self) -> str: """ - Format the scan output data into a plain text string + Format the scan output data into a CycloneDX object """ - return ( - json.dumps(self._normalize_syft_output(), indent=2) - if isinstance(self.scanner.scan_results, SyftScanResult) - else str(self.scanner.scan_results) - ) + cdx = CycloneDx(self.base.debug, self.output_file) + scan_results = {} + for f in self.scanner.decorated_scan_results['files']: + scan_results[f['file']] = [f] + if not cdx.produce_from_json(scan_results, self.output_file): + error_msg = 'Failed to produce CycloneDX output' + self.base.print_stderr(error_msg) + raise ValueError(error_msg) + + def _format_spdxlite_output(self) -> str: + """ + Format the scan output data into a SPDXLite object + """ + spdxlite = SpdxLite(self.base.debug, self.output_file) + scan_results = {} + for f in self.scanner.decorated_scan_results['files']: + scan_results[f['file']] = [f] + if not spdxlite.produce_from_json(scan_results, self.output_file): + error_msg = 'Failed to produce SPDXLite output' + self.base.print_stderr(error_msg) + raise ValueError(error_msg) + + def _format_csv_output(self) -> str: + """ + Format the scan output data into a CSV object + """ + csv = CsvOutput(self.base.debug, self.output_file) + scan_results = {} + for f in self.scanner.decorated_scan_results['files']: + scan_results[f['file']] = [f] + if not csv.produce_from_json(scan_results, self.output_file): + error_msg = 'Failed to produce CSV output' + self.base.print_stderr(error_msg) + raise ValueError(error_msg) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 902f69b7..b3533f8a 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -288,3 +288,12 @@ def _format_plain_output(self) -> str: if isinstance(self.folder_hasher.tree, dict) else str(self.folder_hasher.tree) ) + + def _format_cyclonedx_output(self) -> str: + pass + + def _format_spdxlite_output(self) -> str: + pass + + def _format_csv_output(self) -> str: + pass diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 0af82231..a3937d13 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -158,3 +158,12 @@ def _format_plain_output(self) -> str: if isinstance(self.scanner.scan_results, dict) else str(self.scanner.scan_results) ) + + def _format_cyclonedx_output(self) -> str: + pass + + def _format_spdxlite_output(self) -> str: + pass + + def _format_csv_output(self) -> str: + pass diff --git a/src/scanoss/scanossbase.py b/src/scanoss/scanossbase.py index 64db54cc..de8358ae 100644 --- a/src/scanoss/scanossbase.py +++ b/src/scanoss/scanossbase.py @@ -84,6 +84,9 @@ def print_to_file_or_stdout(self, content: str, file: str = None): """ Print message to file if provided or stdout """ + if not content: + return + if file: with open(file, 'w') as f: f.write(content) @@ -94,6 +97,9 @@ def print_to_file_or_stderr(self, msg: str, file: str = None): """ Print message to file if provided or stderr """ + if not msg: + return + if file: with open(file, 'w') as f: f.write(msg) diff --git a/src/scanoss/utils/abstract_presenter.py b/src/scanoss/utils/abstract_presenter.py index bbc591ee..b7251beb 100644 --- a/src/scanoss/utils/abstract_presenter.py +++ b/src/scanoss/utils/abstract_presenter.py @@ -2,8 +2,6 @@ from scanoss.scanossbase import ScanossBase -AVAILABLE_OUTPUT_FORMATS = ['json', 'plain'] - class AbstractPresenter(ABC): """ @@ -22,6 +20,7 @@ def __init__( """ Initialize the presenter with the given output file and format. """ + self.AVAILABLE_OUTPUT_FORMATS = ['json', 'plain'] self.base = ScanossBase(debug=debug, trace=trace, quiet=quiet) self.output_file = output_file self.output_format = output_format @@ -33,15 +32,21 @@ def present(self, output_format: str = None, output_file: str = None): file_path = output_file or self.output_file fmt = output_format or self.output_format - if fmt and fmt not in AVAILABLE_OUTPUT_FORMATS: + if fmt and fmt not in self.AVAILABLE_OUTPUT_FORMATS: raise ValueError( - f"ERROR: Invalid output format '{fmt}'. Valid values are: {', '.join(AVAILABLE_OUTPUT_FORMATS)}" + f"ERROR: Invalid output format '{fmt}'. Valid values are: {', '.join(self.AVAILABLE_OUTPUT_FORMATS)}" ) if fmt == 'json': content = self._format_json_output() elif fmt == 'plain': content = self._format_plain_output() + elif fmt == 'cyclonedx': + content = self._format_cyclonedx_output() + elif fmt == 'spdxlite': + content = self._format_spdxlite_output() + elif fmt == 'csv': + content = self._format_csv_output() else: content = self._format_plain_output() @@ -53,6 +58,27 @@ def _present_output(self, content: str, file_path: str = None): """ self.base.print_to_file_or_stdout(content, file_path) + @abstractmethod + def _format_cyclonedx_output(self) -> str: + """ + Return a CycloneDX string representation of the data. + """ + pass + + @abstractmethod + def _format_spdxlite_output(self) -> str: + """ + Return a SPDX-Lite string representation of the data. + """ + pass + + @abstractmethod + def _format_csv_output(self) -> str: + """ + Return a CSV string representation of the data. + """ + pass + @abstractmethod def _format_json_output(self) -> str: """ From eb5509b1ec03dd3ca12cbd44bde73acae30271ee Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 31 Mar 2025 14:39:36 +0200 Subject: [PATCH 306/489] feat: ES-213 Format container scanning plain output --- src/scanoss/cli.py | 7 ++++- src/scanoss/scanners/container_scanner.py | 38 ++++++++++++++++++----- src/scanoss/scanners/folder_hasher.py | 3 ++ src/scanoss/scanners/scanner_hfh.py | 3 ++ src/scanoss/utils/abstract_presenter.py | 11 ++++++- 5 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index f61eb880..ebc31f3a 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -599,11 +599,16 @@ def setup_args() -> None: # noqa: PLR0915 # Format options for p in [p_scan, p_cs]: + choices = ['plain', 'cyclonedx', 'spdxlite', 'csv'] + if p is p_cs: + choices.append('raw') + p.add_argument( '--format', '-f', type=str, - choices=['plain', 'cyclonedx', 'spdxlite', 'csv'], + choices=choices, + default='plain', help='Result output format (optional - default: plain)', ) diff --git a/src/scanoss/scanners/container_scanner.py b/src/scanoss/scanners/container_scanner.py index 43fd6048..30d0acd7 100644 --- a/src/scanoss/scanners/container_scanner.py +++ b/src/scanoss/scanners/container_scanner.py @@ -287,9 +287,8 @@ def _execute_syft_scan(self) -> SyftScanResult: SyftExecutionError: If the Syft scan execution fails. """ try: - self.base.print_trace( - f'About to execute {self.syft_command} scan {self.what_to_scan} -q {self.what_to_scan} -o json' - ) + self.base.print_trace(f'About to execute {self.syft_command} scan {self.what_to_scan} -q -o json') + self.base.print_msg('Scanning container...') result = subprocess.run( [self.syft_command, 'scan', self.what_to_scan, '-q', '-o', 'json'], cwd=os.getcwd(), @@ -391,21 +390,38 @@ class ContainerScannerPresenter(AbstractPresenter): def __init__(self, scanner: ContainerScanner, **kwargs): super().__init__(**kwargs) self.scanner = scanner - self.AVAILABLE_OUTPUT_FORMATS = ['plain', 'cyclonedx', 'spdxlite', 'csv'] + self.AVAILABLE_OUTPUT_FORMATS = ['plain', 'cyclonedx', 'spdxlite', 'csv', 'raw'] - def _format_json_output(self) -> str: + def _convert_raw_to_scan_output(self) -> dict: """ - Format the scan output data into a JSON object + Convert the raw output from dependency scanning API to our scan output format Returns: - str: The formatted JSON string + dict: The converted output """ - return json.dumps(self.scanner.decorated_scan_results, indent=2) + formatted_output = {} + if ( + self.scanner.decorated_scan_results + and 'files' in self.scanner.decorated_scan_results + and self.scanner.decorated_scan_results['files'] + and isinstance(self.scanner.decorated_scan_results['files'], list) + ): + file_item = self.scanner.decorated_scan_results['files'][0] + if file_item and isinstance(file_item, dict) and 'file' in file_item: + formatted_output[file_item['file']] = [file_item] + + return formatted_output def _format_plain_output(self) -> str: """ Format the scan output data into a plain text string """ + return json.dumps(self._convert_raw_to_scan_output(), indent=2) + + def _format_raw_output(self) -> str: + """ + Format the scan output data into the raw output from dependency scanning API + """ return json.dumps(self.scanner.decorated_scan_results, indent=2) def _format_cyclonedx_output(self) -> str: @@ -446,3 +462,9 @@ def _format_csv_output(self) -> str: error_msg = 'Failed to produce CSV output' self.base.print_stderr(error_msg) raise ValueError(error_msg) + + def _format_json_output(self) -> str: + """ + Format the scan output data into a JSON object + """ + pass diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index b3533f8a..8dc5c735 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -297,3 +297,6 @@ def _format_spdxlite_output(self) -> str: def _format_csv_output(self) -> str: pass + + def _format_raw_output(self) -> str: + pass diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index a3937d13..a55697ad 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -167,3 +167,6 @@ def _format_spdxlite_output(self) -> str: def _format_csv_output(self) -> str: pass + + def _format_raw_output(self) -> str: + pass diff --git a/src/scanoss/utils/abstract_presenter.py b/src/scanoss/utils/abstract_presenter.py index b7251beb..ba6073db 100644 --- a/src/scanoss/utils/abstract_presenter.py +++ b/src/scanoss/utils/abstract_presenter.py @@ -20,7 +20,7 @@ def __init__( """ Initialize the presenter with the given output file and format. """ - self.AVAILABLE_OUTPUT_FORMATS = ['json', 'plain'] + self.AVAILABLE_OUTPUT_FORMATS = ['json', 'plain', 'cyclonedx', 'spdxlite', 'csv', 'raw'] self.base = ScanossBase(debug=debug, trace=trace, quiet=quiet) self.output_file = output_file self.output_format = output_format @@ -47,6 +47,8 @@ def present(self, output_format: str = None, output_file: str = None): content = self._format_spdxlite_output() elif fmt == 'csv': content = self._format_csv_output() + elif fmt == 'raw': + content = self._format_raw_output() else: content = self._format_plain_output() @@ -92,3 +94,10 @@ def _format_plain_output(self) -> str: Return a plain text string representation of the data. """ pass + + @abstractmethod + def _format_raw_output(self) -> str: + """ + Return a raw string representation of the data. + """ + pass From a7b81d35a30421bd6e3633d1e0f5d6c1f9311f7e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 3 Apr 2025 10:47:28 +0200 Subject: [PATCH 307/489] feat: ES-213 Add container scan raw results to dep command --- src/scanoss/cli.py | 67 ++++++++++++++++------- src/scanoss/results.py | 3 + src/scanoss/scanners/container_scanner.py | 36 ++++++------ 3 files changed, 69 insertions(+), 37 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ebc31f3a..2be695e1 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -195,6 +195,12 @@ def setup_args() -> None: # noqa: PLR0915 help='Scan source code for dependencies, but do not decorate them', ) p_dep.add_argument('scan_loc', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan') + p_dep.add_argument( + '--container', + type=str, + help='Container image to scan. Supports yourrepo/yourimage:tag, Docker tar, ' + 'OCI tar, OCI directory, SIF Container, or generic filesystem directory.', + ) p_dep.add_argument( '--sc-command', type=str, help='Scancode command and path if required (optional - default scancode).' ) @@ -224,27 +230,19 @@ def setup_args() -> None: # noqa: PLR0915 ), ) p_cs.add_argument( - '--retry', '-R', type=int, default=5, help='Retry limit for API communication (optional - default 5)' + '--retry', + '-R', + type=int, + default=DEFAULT_RETRY, + help='Retry limit for API communication (optional - default 5)', ) p_cs.add_argument( '--timeout', '-M', type=int, - default=180, + default=DEFAULT_TIMEOUT, help='Timeout (in seconds) for API communication (optional - default 180)', ) - p_cs.add_argument( - '--syft-command', - type=str, - help='Syft command and path if required (optional - default syft).', - default=DEFAULT_SYFT_COMMAND, - ) - p_cs.add_argument( - '--syft-timeout', - type=int, - default=DEFAULT_SYFT_TIMEOUT, - help='Timeout (in seconds) for syft to complete (optional - default 600)', - ) p_cs.set_defaults(func=container_scan) # Sub-command: file_count @@ -713,6 +711,21 @@ def setup_args() -> None: # noqa: PLR0915 help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times', ) + # Syft options + for p in [p_cs, p_dep]: + p.add_argument( + '--syft-command', + type=str, + help='Syft command and path if required (optional - default syft).', + default=DEFAULT_SYFT_COMMAND, + ) + p.add_argument( + '--syft-timeout', + type=int, + default=DEFAULT_SYFT_TIMEOUT, + help='Timeout (in seconds) for syft to complete (optional - default 600)', + ) + # Help/Trace command options for p in [ p_scan, @@ -1120,11 +1133,16 @@ def dependency(parser, args): args: Namespace Parsed arguments """ - if not args.scan_loc: - print_stderr('Please specify a file/folder') + if not args.scan_loc and not args.container: + print_stderr('Please specify a file/folder or container image') parser.parse_args([args.subparser, '-h']) sys.exit(1) + # Workaround to return syft scan results converted to our dependency output format + if args.container: + args.scan_loc = args.container + return container_scan(parser, args, only_interim_results=True) + if not os.path.exists(args.scan_loc): print_stderr(f'Error: File or folder specified does not exist: {args.scan_loc}.') sys.exit(1) @@ -1723,7 +1741,7 @@ def folder_hash(parser, args): sys.exit(1) -def container_scan(parser, args): +def container_scan(parser, args, only_interim_results: bool = False): """ Run the "container-scan" sub-command Parameters @@ -1741,12 +1759,19 @@ def container_scan(parser, args): sys.exit(1) try: - container_scanner_config = create_container_scanner_config_from_args(args) - container_scanner = ContainerScanner(config=container_scanner_config, what_to_scan=args.scan_loc) + config = create_container_scanner_config_from_args(args) + config.only_interim_results = only_interim_results + container_scanner = ContainerScanner( + config=config, + what_to_scan=args.scan_loc, + ) container_scanner.scan() - container_scanner.decorate_scan_results_with_dependencies() - container_scanner.present(output_file=args.output, output_format=args.format) + if only_interim_results: + container_scanner.present(output_file=config.output, output_format='raw') + else: + container_scanner.decorate_scan_results_with_dependencies() + container_scanner.present(output_file=config.output, output_format=config.format) except Exception as e: print_stderr(f'ERROR: {e}') sys.exit(1) diff --git a/src/scanoss/results.py b/src/scanoss/results.py index b67c980c..df00f465 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -92,6 +92,9 @@ def _format_spdxlite_output(self) -> str: def _format_csv_output(self) -> str: pass + def _format_raw_output(self) -> str: + pass + def _format_plain_output(self) -> str: """Format the output data into a plain text string diff --git a/src/scanoss/scanners/container_scanner.py b/src/scanoss/scanners/container_scanner.py index 30d0acd7..43aec688 100644 --- a/src/scanoss/scanners/container_scanner.py +++ b/src/scanoss/scanners/container_scanner.py @@ -58,26 +58,27 @@ class ContainerScannerConfig: ca_cert: Optional[str] = None syft_command: str = DEFAULT_SYFT_COMMAND syft_timeout: int = DEFAULT_SYFT_TIMEOUT + only_interim_results: bool = False def create_container_scanner_config_from_args(args) -> ContainerScannerConfig: return ContainerScannerConfig( - debug=args.debug, - trace=args.trace, - quiet=args.quiet, - retry=args.retry, - timeout=args.timeout, - output=args.output, - format=args.format, - apiurl=args.api2url, - proxy=args.proxy, - pac=args.pac, - grpc_proxy=args.grpc_proxy, - ca_cert=args.ca_cert, - ignore_cert_errors=args.ignore_cert_errors, - key=args.key, - syft_command=args.syft_command, - syft_timeout=args.syft_timeout, + debug=args.debug if 'debug' in args else False, + trace=args.trace if 'trace' in args else False, + quiet=args.quiet if 'quiet' in args else False, + retry=args.retry if 'retry' in args else DEFAULT_RETRY, + timeout=args.timeout if 'timeout' in args else DEFAULT_TIMEOUT, + output=args.output if 'output' in args else None, + format=args.format if 'format' in args else None, + apiurl=args.api2url if 'api2url' in args else None, + proxy=args.proxy if 'proxy' in args else None, + pac=args.pac if 'pac' in args else None, + grpc_proxy=args.grpc_proxy if 'grpc_proxy' in args else None, + ca_cert=args.ca_cert if 'ca_cert' in args else None, + ignore_cert_errors=args.ignore_cert_errors if 'ignore_cert_errors' in args else False, + key=args.key if 'key' in args else None, + syft_command=args.syft_command if 'syft_command' in args else DEFAULT_SYFT_COMMAND, + syft_timeout=args.syft_timeout if 'syft_timeout' in args else DEFAULT_SYFT_TIMEOUT, ) @@ -234,6 +235,7 @@ def __init__( self.what_to_scan: str = what_to_scan self.syft_command: str = config.syft_command self.syft_timeout: int = config.syft_timeout + self.only_interim_results: bool = config.only_interim_results self.syft_output: Optional[SyftScanResult] = None self.normalized_syft_output: Optional[ContainerScanResult] = None self.decorated_scan_results: Optional[DecoratedContainerScanResult] = None @@ -422,6 +424,8 @@ def _format_raw_output(self) -> str: """ Format the scan output data into the raw output from dependency scanning API """ + if self.scanner.only_interim_results: + return json.dumps(self.scanner.normalized_syft_output, indent=2) return json.dumps(self.scanner.decorated_scan_results, indent=2) def _format_cyclonedx_output(self) -> str: From e4944e563c373a988c3836f6b0cc170941075934 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 23 Apr 2025 11:07:20 +0200 Subject: [PATCH 308/489] feat: ES-213 Fix lint errors --- src/scanoss/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 2be695e1..f7041ee6 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -82,7 +82,7 @@ def print_stderr(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def setup_args() -> None: # noqa: PLR0915 +def setup_args() -> None: # noqa: PLR0912, PLR0915 """ Setup all the command line arguments for processing """ From 470c12712b87ca9fc5bd0caa385910a09362ad59 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 23 Apr 2025 13:53:18 +0200 Subject: [PATCH 309/489] feat: ES-213 Update changelog, client help and docs --- CHANGELOG.md | 10 +++++++- CLIENT_HELP.md | 16 +++++++++++- docs/source/index.rst | 37 ++++++++++++++++++++++++++- src/scanoss/__init__.py | 2 +- src/scanoss/results.py | 8 +++--- src/scanoss/scanners/folder_hasher.py | 8 +++--- src/scanoss/scanners/scanner_hfh.py | 8 +++--- 7 files changed, 73 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c9fbdae..51f88787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.22.0] - 2025-04-23 +### Added +- Add `container-scan` subcommand to scan container images. +- Add `--container` flag to `dependency` subcommand to scan dependencies in container images. +### Modified +- Refactor CLI argument handling for output and format options. + ## [1.21.0] - 2025-03-27 ### Added - Add folder-scan subcommand @@ -498,4 +505,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.4]: https://github.com/scanoss/scanoss.py/compare/v1.20.3...v1.20.4 [1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 [1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 -[1.21.0]: https://github.com/scanoss/scanoss.py/compare/v1.20.6...v1.21.0 \ No newline at end of file +[1.21.0]: https://github.com/scanoss/scanoss.py/compare/v1.20.6...v1.21.0 +[1.22.0]: https://github.com/scanoss/scanoss.py/compare/v1.21.0...v1.22.0 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 8d37fbe8..ed2b7fc2 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -163,6 +163,11 @@ The dependency files of a project can be fingerprinted/parsed using the `dep` co scanoss-py dep -o src-deps.json src ``` +You can also analyze dependencies from a container image using the `--container` flag: +```bash +scanoss-py dep --container ubuntu:latest -o container-deps.json +``` + This parsed dependency file can then be sent to the SCANOSS for decoration using the scanning command: ```bash scanoss-py scan --dep src-deps.json --dependencies-only -o scan-results.json @@ -429,4 +434,13 @@ The new `folder-scan` subcommand performs a comprehensive scan on an entire dire **Usage:** ```shell scanoss-py folder-scan /path/to/folder -o folder-scan-results.json -``` \ No newline at end of file +``` + +### Container-Scan a Docker Image + +The `container-scan` subcommand allows you to scan Docker container images for dependencies. This command extracts and analyzes dependencies from container images, helping you identify open source components within containerized applications. + +**Usage:** +```shell +scanoss-py container-scan image_name:tag -o container-scan-results.json +``` diff --git a/docs/source/index.rst b/docs/source/index.rst index cfc073c7..ba3e8d83 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -174,6 +174,8 @@ Scan source code for dependencies, but do not decorate them. - Description * - --output , -o - Output result file name (optional - default STDOUT) + * - --container + - Analyze dependencies from a Docker container image instead of a directory * - --sc-command SC_COMMAND - Scancode command and path if required (optional - default scancode) * - --sc-timeout SC_TIMEOUT @@ -301,6 +303,39 @@ Both commands also support these general options: * --trace, -t: Enable trace messages * --quiet, -q: Enable quiet mode +------------------------------------ +Container Scanning: container-scan, cs +------------------------------------ + +Scans Docker container images for dependencies, extracting and analyzing components within containerized applications. + +.. code-block:: bash + + scanoss-py container-scan -i + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --image , -i + - Docker image name and tag to scan (required) + * - --output , -o + - Output result file name (optional - default STDOUT) + * - --include-base-image + - Include base image dependencies in the scan results + * - --format , -f + - Output format: {json} (optional - default json) + * - --timeout , -M + - Timeout in seconds for API communication (optional - default 600) + * - --key , -k + - SCANOSS API Key token (optional - not required for default OSSKB URL) + * - --proxy + - Proxy URL to use for connections + * - --ca-cert + - Alternative certificate PEM file + ----------------- Component: ----------------- @@ -434,4 +469,4 @@ The Scanoss Open Source scanoss-py package is released under the MIT license. SCANOSS Website GitHub - Software transparency foundation \ No newline at end of file + Software transparency foundation diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index c18db761..e7a2405b 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.21.0' +__version__ = '1.22.0' diff --git a/src/scanoss/results.py b/src/scanoss/results.py index df00f465..5cbff282 100644 --- a/src/scanoss/results.py +++ b/src/scanoss/results.py @@ -84,16 +84,16 @@ def _format_json_output(self) -> str: return '' def _format_cyclonedx_output(self) -> str: - pass + raise NotImplementedError('CycloneDX output is not implemented') def _format_spdxlite_output(self) -> str: - pass + raise NotImplementedError('SPDXlite output is not implemented') def _format_csv_output(self) -> str: - pass + raise NotImplementedError('CSV output is not implemented') def _format_raw_output(self) -> str: - pass + raise NotImplementedError('Raw output is not implemented') def _format_plain_output(self) -> str: """Format the output data into a plain text string diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 8dc5c735..f8654859 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -290,13 +290,13 @@ def _format_plain_output(self) -> str: ) def _format_cyclonedx_output(self) -> str: - pass + raise NotImplementedError('CycloneDX output is not implemented') def _format_spdxlite_output(self) -> str: - pass + raise NotImplementedError('SPDXlite output is not implemented') def _format_csv_output(self) -> str: - pass + raise NotImplementedError('CSV output is not implemented') def _format_raw_output(self) -> str: - pass + raise NotImplementedError('Raw output is not implemented') diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index a55697ad..ea5edbbb 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -160,13 +160,13 @@ def _format_plain_output(self) -> str: ) def _format_cyclonedx_output(self) -> str: - pass + raise NotImplementedError('CycloneDX output is not implemented') def _format_spdxlite_output(self) -> str: - pass + raise NotImplementedError('SPDXlite output is not implemented') def _format_csv_output(self) -> str: - pass + raise NotImplementedError('CSV output is not implemented') def _format_raw_output(self) -> str: - pass + raise NotImplementedError('Raw output is not implemented') From b5aef8d7ab5fe6f25636f4f2d9250fd77e802369 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 23 Apr 2025 14:06:08 +0200 Subject: [PATCH 310/489] feat: ES-213 Fix wfp scanoss settings bug --- CHANGELOG.md | 2 ++ src/scanoss/cli.py | 2 +- src/scanoss/scanoss_settings.py | 14 +++++++++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51f88787..cbc41834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add `--container` flag to `dependency` subcommand to scan dependencies in container images. ### Modified - Refactor CLI argument handling for output and format options. +### Fixed +- Fixed issue with wfp command where settings file was being loaded from the cwd instead of the scan root directory ## [1.21.0] - 2025-03-27 ### Added diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index f7041ee6..29a10150 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -844,7 +844,7 @@ def wfp(parser, args): if not args.skip_settings_file: scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) try: - scan_settings.load_json_file(args.settings) + scan_settings.load_json_file(args.settings, args.scan_dir) except ScanossSettingsError as e: print_stderr(f'Error: {e}') sys.exit(1) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 0903f98f..1770697d 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -24,13 +24,17 @@ import json from pathlib import Path -from typing import List, TypedDict +from typing import List, Optional, TypedDict import importlib_resources from jsonschema import validate from .scanossbase import ScanossBase -from .utils.file import JSON_ERROR_FILE_NOT_FOUND, JSON_ERROR_FILE_EMPTY, validate_json_file +from .utils.file import ( + JSON_ERROR_FILE_EMPTY, + JSON_ERROR_FILE_NOT_FOUND, + validate_json_file, +) DEFAULT_SCANOSS_JSON_FILE = Path('scanoss.json') @@ -96,7 +100,7 @@ def __init__( if filepath: self.load_json_file(filepath) - def load_json_file(self, filepath: 'str | None' = None, scan_root: 'str | None' = None) -> 'ScanossSettings': + def load_json_file(self, filepath: Optional[str] = None, scan_root: Optional[str] = None) -> 'ScanossSettings': """ Load the scan settings file. If no filepath is provided, scanoss.json will be used as default. @@ -118,7 +122,7 @@ def load_json_file(self, filepath: 'str | None' = None, scan_root: 'str | None' result = validate_json_file(json_file) if not result.is_valid: - if result.error_code == JSON_ERROR_FILE_NOT_FOUND or result.error_code == JSON_ERROR_FILE_EMPTY: + if result.error_code in (JSON_ERROR_FILE_NOT_FOUND, JSON_ERROR_FILE_EMPTY): self.print_msg( f'WARNING: The supplied settings file "{filepath}" was not found or is empty. Skipping...' ) @@ -235,7 +239,7 @@ def _get_sbom_assets(self): include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include())) replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace())) self.print_debug( - f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n" + f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n" # noqa: E501 f'From Include list: {[entry["purl"] for entry in include_bom_entries]} \n' f'From Replace list: {[entry["purl"] for entry in replace_bom_entries]} \n' ) From 70bd84fbf8c22bb907370ee9ebe18c1083eb79e0 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 23 Apr 2025 16:39:27 +0200 Subject: [PATCH 311/489] feat: ES-213 Fix provenance command --- src/scanoss/cli.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 29a10150..d37d5d54 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -667,7 +667,17 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.') # Global Scan/GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, p_folder_scan, p_cs]: + for p in [ + p_scan, + c_crypto, + c_vulns, + c_search, + c_versions, + c_semgrep, + c_provenance, + p_folder_scan, + p_cs, + ]: p.add_argument( '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' ) @@ -693,7 +703,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, p_folder_scan, p_cs]: + for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan, p_cs]: p.add_argument( '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' ) From f2edeade5dca3da1229868abe2634d56967f4555 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Thu, 24 Apr 2025 14:26:43 +0100 Subject: [PATCH 312/489] update geoprovenance --- .../{provenance => geoprovenance}/__init__.py | 0 .../v2/__init__.py | 0 .../v2/scanoss_geoprovenance_pb2.py | 49 ++++++ .../v2/scanoss_geoprovenance_pb2_grpc.py | 142 ++++++++++++++++++ .../provenance/v2/scanoss_provenance_pb2.py | 41 ----- .../v2/scanoss_provenance_pb2_grpc.py | 108 ------------- 6 files changed, 191 insertions(+), 149 deletions(-) rename src/scanoss/api/{provenance => geoprovenance}/__init__.py (100%) rename src/scanoss/api/{provenance => geoprovenance}/v2/__init__.py (100%) create mode 100644 src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py create mode 100644 src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py delete mode 100644 src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py delete mode 100644 src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py diff --git a/src/scanoss/api/provenance/__init__.py b/src/scanoss/api/geoprovenance/__init__.py similarity index 100% rename from src/scanoss/api/provenance/__init__.py rename to src/scanoss/api/geoprovenance/__init__.py diff --git a/src/scanoss/api/provenance/v2/__init__.py b/src/scanoss/api/geoprovenance/v2/__init__.py similarity index 100% rename from src/scanoss/api/provenance/v2/__init__.py rename to src/scanoss/api/geoprovenance/v2/__init__.py diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py new file mode 100644 index 00000000..cdd924cc --- /dev/null +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto\x12\x19scanoss.api.provenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc8\x03\n\x13\x43ontributorResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.provenance.v2.ContributorResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xcd\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12[\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32?.scanoss.api.provenance.v2.ContributorResponse.DeclaredLocation\x12Y\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32>.scanoss.api.provenance.v2.ContributorResponse.CuratedLocation\"\x93\x02\n\x0eOriginResponse\x12>\n\x05purls\x18\x01 \x03(\x0b\x32/.scanoss.api.provenance.v2.OriginResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a,\n\x08Location\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\npercentage\x18\x02 \x01(\x02\x1a\\\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x45\n\tlocations\x18\x02 \x03(\x0b\x32\x32.scanoss.api.provenance.v2.OriginResponse.Location2\xb3\x03\n\rGeoProvenance\x12v\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/geoprovenance/echo:\x01*\x12\x9a\x01\n\x18GetComponentContributors\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.provenance.v2.ContributorResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/geoprovenance/countries:\x01*\x12\x8c\x01\n\x12GetComponentOrigin\x12\".scanoss.api.common.v2.PurlRequest\x1a).scanoss.api.provenance.v2.OriginResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/geoprovenance/origin:\x01*B\xa4\x02Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\x92\x41\xe3\x01\x12}\n\x1eSCANOSS GEO Provenance Service\"V\n\x15scanoss-geoprovenance\x12(https://github.com/scanoss/geoprovenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.geoprovenance.v2.scanoss_geoprovenance_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\222A\343\001\022}\n\036SCANOSS GEO Provenance Service\"V\n\025scanoss-geoprovenance\022(https://github.com/scanoss/geoprovenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _GEOPROVENANCE.methods_by_name['Echo']._options = None + _GEOPROVENANCE.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/geoprovenance/echo:\001*' + _GEOPROVENANCE.methods_by_name['GetComponentContributors']._options = None + _GEOPROVENANCE.methods_by_name['GetComponentContributors']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/geoprovenance/countries:\001*' + _GEOPROVENANCE.methods_by_name['GetComponentOrigin']._options = None + _GEOPROVENANCE.methods_by_name['GetComponentOrigin']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/geoprovenance/origin:\001*' + _CONTRIBUTORRESPONSE._serialized_start=208 + _CONTRIBUTORRESPONSE._serialized_end=664 + _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_start=355 + _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_end=405 + _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_start=407 + _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_end=456 + _CONTRIBUTORRESPONSE_PURLS._serialized_start=459 + _CONTRIBUTORRESPONSE_PURLS._serialized_end=664 + _ORIGINRESPONSE._serialized_start=667 + _ORIGINRESPONSE._serialized_end=942 + _ORIGINRESPONSE_LOCATION._serialized_start=804 + _ORIGINRESPONSE_LOCATION._serialized_end=848 + _ORIGINRESPONSE_PURLS._serialized_start=850 + _ORIGINRESPONSE_PURLS._serialized_end=942 + _GEOPROVENANCE._serialized_start=945 + _GEOPROVENANCE._serialized_end=1380 +# @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py new file mode 100644 index 00000000..b0c93821 --- /dev/null +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py @@ -0,0 +1,142 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from scanoss.api.geoprovenance.v2 import scanoss_geoprovenance_pb2 as scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2 + + +class GeoProvenanceStub(object): + """* + Expose all of the SCANOSS Geo Provenance RPCs here + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Echo = channel.unary_unary( + '/scanoss.api.provenance.v2.GeoProvenance/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + ) + self.GetComponentContributors = channel.unary_unary( + '/scanoss.api.provenance.v2.GeoProvenance/GetComponentContributors', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.FromString, + ) + self.GetComponentOrigin = channel.unary_unary( + '/scanoss.api.provenance.v2.GeoProvenance/GetComponentOrigin', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.FromString, + ) + + +class GeoProvenanceServicer(object): + """* + Expose all of the SCANOSS Geo Provenance RPCs here + """ + + def Echo(self, request, context): + """Standard echo + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentContributors(self, request, context): + """Get component-level Geo Provenance based on contributor declared location + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentOrigin(self, request, context): + """Get component-level Geo Provenance based on contributor origin commit times + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_GeoProvenanceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetComponentContributors': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentContributors, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.SerializeToString, + ), + 'GetComponentOrigin': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentOrigin, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, + response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.provenance.v2.GeoProvenance', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class GeoProvenance(object): + """* + Expose all of the SCANOSS Geo Provenance RPCs here + """ + + @staticmethod + def Echo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.GeoProvenance/Echo', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetComponentContributors(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.GeoProvenance/GetComponentContributors', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetComponentOrigin(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.GeoProvenance/GetComponentOrigin', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, + scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py deleted file mode 100644 index 5af1b6b2..00000000 --- a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: scanoss/api/provenance/v2/scanoss-provenance.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/provenance/v2/scanoss-provenance.proto\x12\x19scanoss.api.provenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xd5\x03\n\x12ProvenanceResponse\x12\x42\n\x05purls\x18\x01 \x03(\x0b\x32\x33.scanoss.api.provenance.v2.ProvenanceResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xdc\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12Z\n\x12\x64\x65\x63lared_locations\x18\x03 \x03(\x0b\x32>.scanoss.api.provenance.v2.ProvenanceResponse.DeclaredLocation\x12X\n\x11\x63urated_locations\x18\x04 \x03(\x0b\x32=.scanoss.api.provenance.v2.ProvenanceResponse.CuratedLocation2\x98\x02\n\nProvenance\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/provenance/echo:\x01*\x12\x94\x01\n\x16GetComponentProvenance\x12\".scanoss.api.common.v2.PurlRequest\x1a-.scanoss.api.provenance.v2.ProvenanceResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/provenance/countries:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/provenancev2;provenancev2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Provenance Service\"P\n\x12scanoss-provenance\x12%https://github.com/scanoss/provenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.provenance.v2.scanoss_provenance_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/provenancev2;provenancev2\222A\331\001\022s\n\032SCANOSS Provenance Service\"P\n\022scanoss-provenance\022%https://github.com/scanoss/provenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _PROVENANCE.methods_by_name['Echo']._options = None - _PROVENANCE.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/provenance/echo:\001*' - _PROVENANCE.methods_by_name['GetComponentProvenance']._options = None - _PROVENANCE.methods_by_name['GetComponentProvenance']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/provenance/countries:\001*' - _PROVENANCERESPONSE._serialized_start=202 - _PROVENANCERESPONSE._serialized_end=671 - _PROVENANCERESPONSE_DECLAREDLOCATION._serialized_start=347 - _PROVENANCERESPONSE_DECLAREDLOCATION._serialized_end=397 - _PROVENANCERESPONSE_CURATEDLOCATION._serialized_start=399 - _PROVENANCERESPONSE_CURATEDLOCATION._serialized_end=448 - _PROVENANCERESPONSE_PURLS._serialized_start=451 - _PROVENANCERESPONSE_PURLS._serialized_end=671 - _PROVENANCE._serialized_start=674 - _PROVENANCE._serialized_end=954 -# @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py b/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py deleted file mode 100644 index 1be8745d..00000000 --- a/src/scanoss/api/provenance/v2/scanoss_provenance_pb2_grpc.py +++ /dev/null @@ -1,108 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 -from scanoss.api.provenance.v2 import scanoss_provenance_pb2 as scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2 - - -class ProvenanceStub(object): - """* - Expose all of the SCANOSS Provenance RPCs here - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Echo = channel.unary_unary( - '/scanoss.api.provenance.v2.Provenance/Echo', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) - self.GetComponentProvenance = channel.unary_unary( - '/scanoss.api.provenance.v2.Provenance/GetComponentProvenance', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2.ProvenanceResponse.FromString, - ) - - -class ProvenanceServicer(object): - """* - Expose all of the SCANOSS Provenance RPCs here - """ - - def Echo(self, request, context): - """Standard echo - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetComponentProvenance(self, request, context): - """Get Provenance countries associated with a list of PURLs - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_ProvenanceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Echo': grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, - response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, - ), - 'GetComponentProvenance': grpc.unary_unary_rpc_method_handler( - servicer.GetComponentProvenance, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2.ProvenanceResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.provenance.v2.Provenance', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class Provenance(object): - """* - Expose all of the SCANOSS Provenance RPCs here - """ - - @staticmethod - def Echo(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.Provenance/Echo', - scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, - scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def GetComponentProvenance(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.Provenance/GetComponentProvenance', - scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - scanoss_dot_api_dot_provenance_dot_v2_dot_scanoss__provenance__pb2.ProvenanceResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From b35b4bf17d4a27f7b00f198cd12fa4b412868679 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 24 Apr 2025 15:59:54 +0200 Subject: [PATCH 313/489] feat: SP-2388 Update provenance command, add --origin argument, update stubs --- src/scanoss/cli.py | 3 ++- src/scanoss/components.py | 32 +++++++++++++++++++++----------- src/scanoss/scanossgrpc.py | 27 ++++++++++++++++++++++----- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index d37d5d54..0ad90d5b 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -326,6 +326,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 description=f'Show Provenance findings: {__version__}', help='Retrieve provenance for the given components', ) + c_provenance.add_argument('--origin', action='store_true', help='Retrieve provenance using contributors origin') c_provenance.set_defaults(func=comp_provenance) # Component Sub-command: component search @@ -1608,7 +1609,7 @@ def comp_provenance(parser, args): timeout=args.timeout, req_headers=process_req_headers(args.header), ) - if not comps.get_provenance_details(args.input, args.purl, args.output): + if not comps.get_provenance_details(args.input, args.purl, args.output, args.origin): sys.exit(1) diff --git a/src/scanoss/components.py b/src/scanoss/components.py index fcecec56..f054e488 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -39,7 +39,7 @@ class Components(ScanossBase): Class for Component functionality """ - def __init__( # noqa: PLR0913, PLR0915 + def __init__( # noqa: PLR0913, PLR0915 self, debug: bool = False, trace: bool = False, @@ -244,7 +244,7 @@ def get_semgrep_details(self, json_file: str = None, purls: [] = None, output_fi self._close_file(output_file, file) return success - def search_components( # noqa: PLR0913, PLR0915 + def search_components( # noqa: PLR0913, PLR0915 self, output_file: str = None, json_file: str = None, @@ -330,14 +330,20 @@ def get_component_versions( self._close_file(output_file, file) return success - def get_provenance_details(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: + def get_provenance_details( + self, json_file: str = None, purls: [] = None, output_file: str = None, origin: bool = False + ) -> bool: """ - Retrieve the semgrep details for the supplied PURLs + Retrieve the provenance details for the supplied PURLs - :param json_file: PURL JSON request file (optional) - :param purls: PURL request array (optional) - :param output_file: output filename (optional). Default: STDOUT - :return: True on success, False otherwise + Args: + json_file (str, optional): Input JSON file. Defaults to None. + purls (None, optional): PURLs to retrieve provenance details for. Defaults to None. + output_file (str, optional): Output file. Defaults to None. + origin (bool, optional): Retrieve origin details. Defaults to False. + + Returns: + bool: True on success, False otherwise """ success = False purls_request = self.load_purls(json_file, purls) @@ -346,12 +352,16 @@ def get_provenance_details(self, json_file: str = None, purls: [] = None, output file = self._open_file_or_sdtout(output_file) if file is None: return False - self.print_msg('Sending PURLs to Provenance API for decoration...') - response = self.grpc_api.get_provenance_json(purls_request) + if origin: + self.print_msg('Sending PURLs to Geo Provenance API for decoration...') + response = self.grpc_api.get_provenance_origin(purls_request) + else: + self.print_msg('Sending PURLs to Provenance API for decoration...') + response = self.grpc_api.get_provenance_json(purls_request) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) success = True if output_file: self.print_msg(f'Results written to: {output_file}') self._close_file(output_file, file) - return success \ No newline at end of file + return success diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 9271d7eb..fea1e33e 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -36,7 +36,6 @@ from pypac.parser import PACFile from pypac.resolver import ProxyResolver -from scanoss.api.provenance.v2.scanoss_provenance_pb2_grpc import ProvenanceStub from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub from scanoss.constants import DEFAULT_TIMEOUT @@ -59,7 +58,8 @@ from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub -from .api.provenance.v2.scanoss_provenance_pb2 import ProvenanceResponse +from .api.geoprovenance.v2.scanoss_geoprovenance_pb2 import ContributorResponse +from .api.geoprovenance.v2.scanoss_geoprovenance_pb2_grpc import GeoProvenanceStub from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub @@ -169,7 +169,7 @@ def __init__( # noqa: PLR0913, PLR0915 self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) self.semgrep_stub = SemgrepStub(grpc.insecure_channel(self.url)) self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url)) - self.provenance_stub = ProvenanceStub(grpc.insecure_channel(self.url)) + self.provenance_stub = GeoProvenanceStub(grpc.insecure_channel(self.url)) self.scanning_stub = ScanningStub(grpc.insecure_channel(self.url)) else: if ca_cert is not None: @@ -181,7 +181,7 @@ def __init__( # noqa: PLR0913, PLR0915 self.dependencies_stub = DependenciesStub(grpc.secure_channel(self.url, credentials)) self.semgrep_stub = SemgrepStub(grpc.secure_channel(self.url, credentials)) self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials)) - self.provenance_stub = ProvenanceStub(grpc.secure_channel(self.url, credentials)) + self.provenance_stub = GeoProvenanceStub(grpc.secure_channel(self.url, credentials)) self.scanning_stub = ScanningStub(grpc.secure_channel(self.url, credentials)) @classmethod @@ -574,7 +574,7 @@ def get_provenance_json(self, purls: dict) -> dict: self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) - resp: ProvenanceResponse + resp: ContributorResponse try: request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object metadata = self.metadata[:] @@ -593,6 +593,23 @@ def get_provenance_json(self, purls: dict) -> dict: return resp_dict return None + def get_provenance_origin(self, request: Dict) -> Dict: + """ + Client function to call the rpc for GetComponentOrigin + + Args: + request (Dict): GetComponentOrigin Request + + Returns: + Dict: OriginResponse + """ + return self._call_rpc( + self.provenance_stub.GetComponentOrigin, + request, + PurlRequest, + 'Sending data for provenance origin decoration (rqId: {rqId})...', + ) + def load_generic_headers(self): """ Adds custom headers from req_headers to metadata. From 86fc0d920328ee6241cec005cd142a635d5b347e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 24 Apr 2025 16:02:27 +0200 Subject: [PATCH 314/489] feat: SP-2388 Update changelog and version --- CHANGELOG.md | 9 ++++++++- CLIENT_HELP.md | 7 +++++++ src/scanoss/__init__.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc41834..9be3376a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.22.1] - 2025-04-24 +### Added +- Add `--origin` flag to `component provenance` subcommand to retrieve provenance using contributors origin +### Modified +- Update provenance GRPC stubs + ## [1.22.0] - 2025-04-23 ### Added - Add `container-scan` subcommand to scan container images. @@ -508,4 +514,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.5]: https://github.com/scanoss/scanoss.py/compare/v1.20.4...v1.20.5 [1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 [1.21.0]: https://github.com/scanoss/scanoss.py/compare/v1.20.6...v1.21.0 -[1.22.0]: https://github.com/scanoss/scanoss.py/compare/v1.21.0...v1.22.0 \ No newline at end of file +[1.22.0]: https://github.com/scanoss/scanoss.py/compare/v1.21.0...v1.22.0 +[1.22.1]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.22.1 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index ed2b7fc2..34e9d824 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -329,6 +329,13 @@ scanoss-py comp prov -p "pkg:github/unoconv/unoconv" It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): ```bash scanoss-py comp prov -i purl-input.json -o provenance.json +``` + +#### Component Provenance Using Contributors Origin +The following command provides the capability to search the SCANOSS KB for component Provenance using contributors origin: +```bash +scanoss-py comp prov -p "pkg:github/unoconv/unoconv" --origin +``` ### Results Commands diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index e7a2405b..324704aa 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.22.0' +__version__ = '1.22.1' From 4ab31a9be945d82f7d239d5146d16d0f94f561ce Mon Sep 17 00:00:00 2001 From: eeisegn Date: Thu, 24 Apr 2025 18:37:05 +0100 Subject: [PATCH 315/489] fix function grpc call --- .../v2/scanoss_geoprovenance_pb2.py | 34 +++++++++---------- .../v2/scanoss_geoprovenance_pb2_grpc.py | 14 ++++---- src/scanoss/scanossgrpc.py | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py index cdd924cc..df9e5754 100644 --- a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py @@ -16,7 +16,7 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto\x12\x19scanoss.api.provenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc8\x03\n\x13\x43ontributorResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.provenance.v2.ContributorResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xcd\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12[\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32?.scanoss.api.provenance.v2.ContributorResponse.DeclaredLocation\x12Y\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32>.scanoss.api.provenance.v2.ContributorResponse.CuratedLocation\"\x93\x02\n\x0eOriginResponse\x12>\n\x05purls\x18\x01 \x03(\x0b\x32/.scanoss.api.provenance.v2.OriginResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a,\n\x08Location\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\npercentage\x18\x02 \x01(\x02\x1a\\\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x45\n\tlocations\x18\x02 \x03(\x0b\x32\x32.scanoss.api.provenance.v2.OriginResponse.Location2\xb3\x03\n\rGeoProvenance\x12v\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/geoprovenance/echo:\x01*\x12\x9a\x01\n\x18GetComponentContributors\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.provenance.v2.ContributorResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/geoprovenance/countries:\x01*\x12\x8c\x01\n\x12GetComponentOrigin\x12\".scanoss.api.common.v2.PurlRequest\x1a).scanoss.api.provenance.v2.OriginResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/geoprovenance/origin:\x01*B\xa4\x02Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\x92\x41\xe3\x01\x12}\n\x1eSCANOSS GEO Provenance Service\"V\n\x15scanoss-geoprovenance\x12(https://github.com/scanoss/geoprovenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto\x12\x1cscanoss.api.geoprovenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xd1\x03\n\x13\x43ontributorResponse\x12\x46\n\x05purls\x18\x01 \x03(\x0b\x32\x37.scanoss.api.geoprovenance.v2.ContributorResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xd3\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12^\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32\x42.scanoss.api.geoprovenance.v2.ContributorResponse.DeclaredLocation\x12\\\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32\x41.scanoss.api.geoprovenance.v2.ContributorResponse.CuratedLocation\"\x99\x02\n\x0eOriginResponse\x12\x41\n\x05purls\x18\x01 \x03(\x0b\x32\x32.scanoss.api.geoprovenance.v2.OriginResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a,\n\x08Location\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\npercentage\x18\x02 \x01(\x02\x1a_\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12H\n\tlocations\x18\x02 \x03(\x0b\x32\x35.scanoss.api.geoprovenance.v2.OriginResponse.Location2\xb9\x03\n\rGeoProvenance\x12v\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/geoprovenance/echo:\x01*\x12\x9d\x01\n\x18GetComponentContributors\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.geoprovenance.v2.ContributorResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/geoprovenance/countries:\x01*\x12\x8f\x01\n\x12GetComponentOrigin\x12\".scanoss.api.common.v2.PurlRequest\x1a,.scanoss.api.geoprovenance.v2.OriginResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/geoprovenance/origin:\x01*B\xa4\x02Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\x92\x41\xe3\x01\x12}\n\x1eSCANOSS GEO Provenance Service\"V\n\x15scanoss-geoprovenance\x12(https://github.com/scanoss/geoprovenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.geoprovenance.v2.scanoss_geoprovenance_pb2', globals()) @@ -30,20 +30,20 @@ _GEOPROVENANCE.methods_by_name['GetComponentContributors']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/geoprovenance/countries:\001*' _GEOPROVENANCE.methods_by_name['GetComponentOrigin']._options = None _GEOPROVENANCE.methods_by_name['GetComponentOrigin']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/geoprovenance/origin:\001*' - _CONTRIBUTORRESPONSE._serialized_start=208 - _CONTRIBUTORRESPONSE._serialized_end=664 - _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_start=355 - _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_end=405 - _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_start=407 - _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_end=456 - _CONTRIBUTORRESPONSE_PURLS._serialized_start=459 - _CONTRIBUTORRESPONSE_PURLS._serialized_end=664 - _ORIGINRESPONSE._serialized_start=667 - _ORIGINRESPONSE._serialized_end=942 - _ORIGINRESPONSE_LOCATION._serialized_start=804 - _ORIGINRESPONSE_LOCATION._serialized_end=848 - _ORIGINRESPONSE_PURLS._serialized_start=850 - _ORIGINRESPONSE_PURLS._serialized_end=942 - _GEOPROVENANCE._serialized_start=945 - _GEOPROVENANCE._serialized_end=1380 + _CONTRIBUTORRESPONSE._serialized_start=211 + _CONTRIBUTORRESPONSE._serialized_end=676 + _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_start=361 + _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_end=411 + _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_start=413 + _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_end=462 + _CONTRIBUTORRESPONSE_PURLS._serialized_start=465 + _CONTRIBUTORRESPONSE_PURLS._serialized_end=676 + _ORIGINRESPONSE._serialized_start=679 + _ORIGINRESPONSE._serialized_end=960 + _ORIGINRESPONSE_LOCATION._serialized_start=819 + _ORIGINRESPONSE_LOCATION._serialized_end=863 + _ORIGINRESPONSE_PURLS._serialized_start=865 + _ORIGINRESPONSE_PURLS._serialized_end=960 + _GEOPROVENANCE._serialized_start=963 + _GEOPROVENANCE._serialized_end=1404 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py index b0c93821..ff63832a 100644 --- a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py @@ -18,17 +18,17 @@ def __init__(self, channel): channel: A grpc.Channel. """ self.Echo = channel.unary_unary( - '/scanoss.api.provenance.v2.GeoProvenance/Echo', + '/scanoss.api.geoprovenance.v2.GeoProvenance/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, ) self.GetComponentContributors = channel.unary_unary( - '/scanoss.api.provenance.v2.GeoProvenance/GetComponentContributors', + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentContributors', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.FromString, ) self.GetComponentOrigin = channel.unary_unary( - '/scanoss.api.provenance.v2.GeoProvenance/GetComponentOrigin', + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentOrigin', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.FromString, ) @@ -80,7 +80,7 @@ def add_GeoProvenanceServicer_to_server(servicer, server): ), } generic_handler = grpc.method_handlers_generic_handler( - 'scanoss.api.provenance.v2.GeoProvenance', rpc_method_handlers) + 'scanoss.api.geoprovenance.v2.GeoProvenance', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) @@ -101,7 +101,7 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.GeoProvenance/Echo', + return grpc.experimental.unary_unary(request, target, '/scanoss.api.geoprovenance.v2.GeoProvenance/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, options, channel_credentials, @@ -118,7 +118,7 @@ def GetComponentContributors(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.GeoProvenance/GetComponentContributors', + return grpc.experimental.unary_unary(request, target, '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentContributors', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.FromString, options, channel_credentials, @@ -135,7 +135,7 @@ def GetComponentOrigin(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.provenance.v2.GeoProvenance/GetComponentOrigin', + return grpc.experimental.unary_unary(request, target, '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentOrigin', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.FromString, options, channel_credentials, diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index fea1e33e..02c73077 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -580,7 +580,7 @@ def get_provenance_json(self, purls: dict) -> dict: metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...') - resp = self.provenance_stub.GetComponentProvenance(request, metadata=metadata, timeout=self.timeout) + resp = self.provenance_stub.GetComponentContributors(request, metadata=metadata, timeout=self.timeout) except Exception as e: self.print_stderr( f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' From 0015c487edd56ac57472c850efd916ca7eb4bb63 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 25 Apr 2025 08:47:10 +0200 Subject: [PATCH 316/489] feat: SP-2388 Add status to grpc response --- src/scanoss/scanossgrpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 02c73077..da3fabfc 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -515,7 +515,6 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional raise ScanossGrpcError(f'Unsuccessful status response (rqId: {request_id}).') resp_dict = MessageToDict(resp, preserving_proto_field_name=True) - resp_dict.pop('status', None) return resp_dict def _check_status_response(self, status_response: StatusResponse, request_id: str = None) -> bool: From 2badd6ff3461b2c2ed4ffb4b339726d926957e9b Mon Sep 17 00:00:00 2001 From: eeisegn Date: Fri, 25 Apr 2025 09:08:23 +0100 Subject: [PATCH 317/489] text cleanup --- src/scanoss/cli.py | 8 ++++---- src/scanoss/components.py | 4 ++-- src/scanoss/scanossgrpc.py | 2 -- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 0ad90d5b..866d3bd6 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -323,10 +323,10 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_provenance = comp_sub.add_parser( 'provenance', aliases=['prov', 'prv'], - description=f'Show Provenance findings: {__version__}', - help='Retrieve provenance for the given components', + description=f'Show GEO Provenance findings: {__version__}', + help='Retrieve geoprovenance for the given components', ) - c_provenance.add_argument('--origin', action='store_true', help='Retrieve provenance using contributors origin') + c_provenance.add_argument('--origin', action='store_true', help='Retrieve geoprovenance using contributors origin (default: declared origin)') c_provenance.set_defaults(func=comp_provenance) # Component Sub-command: component search @@ -1580,7 +1580,7 @@ def comp_versions(parser, args): def comp_provenance(parser, args): """ - Run the "component semgrep" sub-command + Run the "component provenance" sub-command Parameters ---------- parser: ArgumentParser diff --git a/src/scanoss/components.py b/src/scanoss/components.py index f054e488..c68a2336 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -353,10 +353,10 @@ def get_provenance_details( if file is None: return False if origin: - self.print_msg('Sending PURLs to Geo Provenance API for decoration...') + self.print_msg('Sending PURLs to Geo Provenance Origin API for decoration...') response = self.grpc_api.get_provenance_origin(purls_request) else: - self.print_msg('Sending PURLs to Provenance API for decoration...') + self.print_msg('Sending PURLs to Geo Provenance Declared API for decoration...') response = self.grpc_api.get_provenance_json(purls_request) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index da3fabfc..e2e4a451 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -492,7 +492,6 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional Returns: dict: The parsed gRPC response as a dictionary, or None if an error occurred. """ - request_id = str(uuid.uuid4()) if isinstance(request_input, dict): @@ -503,7 +502,6 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional metadata = self.metadata[:] + [('x-request-id', request_id)] self.print_debug(debug_msg.format(rqId=request_id)) - try: resp = rpc_method(request_obj, metadata=metadata, timeout=self.timeout) except grpc.RpcError as e: From 15f3df5ad86a34c41eeb66fb671f73495a849e5b Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 25 Apr 2025 10:21:43 +0200 Subject: [PATCH 318/489] feat: SP-2388 Do not throw gRPC exception if request succeeded --- CHANGELOG.md | 4 ++-- src/scanoss/cli.py | 10 +++++++--- src/scanoss/scanners/scanner_hfh.py | 3 ++- src/scanoss/scanossgrpc.py | 14 +++++++------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9be3376a..40c997f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.22.1] - 2025-04-24 +## [1.23.0] - 2025-04-24 ### Added - Add `--origin` flag to `component provenance` subcommand to retrieve provenance using contributors origin ### Modified @@ -515,4 +515,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 [1.21.0]: https://github.com/scanoss/scanoss.py/compare/v1.20.6...v1.21.0 [1.22.0]: https://github.com/scanoss/scanoss.py/compare/v1.21.0...v1.22.0 -[1.22.1]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.22.1 \ No newline at end of file +[1.23.0]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.23.0 \ No newline at end of file diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 866d3bd6..00a97c62 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -326,7 +326,11 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 description=f'Show GEO Provenance findings: {__version__}', help='Retrieve geoprovenance for the given components', ) - c_provenance.add_argument('--origin', action='store_true', help='Retrieve geoprovenance using contributors origin (default: declared origin)') + c_provenance.add_argument( + '--origin', + action='store_true', + help='Retrieve geoprovenance using contributors origin (default: declared origin)', + ) c_provenance.set_defaults(func=comp_provenance) # Component Sub-command: component search @@ -1712,8 +1716,8 @@ def folder_hashing_scan(parser, args): scanner.best_match = args.best_match scanner.threshold = args.threshold - scanner.scan() - scanner.present(output_file=args.output, output_format=args.format) + if scanner.scan(): + scanner.present(output_file=args.output, output_format=args.format) except ScanossGrpcError as e: print_stderr(f'ERROR: {e}') sys.exit(1) diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index ea5edbbb..4b573845 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -117,7 +117,8 @@ def spin(): try: response = self.client.folder_hash_scan(hfh_request) - self.scan_results = response + if response: + self.scan_results = response finally: stop_spinner = True spinner_thread.join() diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index e2e4a451..17752f4d 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -462,7 +462,7 @@ def get_component_versions_json(self, search: dict) -> dict: return resp_dict return None - def folder_hash_scan(self, request: Dict) -> Dict: + def folder_hash_scan(self, request: Dict) -> Optional[Dict]: """ Client function to call the rpc for Folder Hashing Scan @@ -470,7 +470,7 @@ def folder_hash_scan(self, request: Dict) -> Dict: request (Dict): Folder Hash Request Returns: - Dict: Folder Hash Response + Optional[Dict]: Folder Hash Response, or None if the request was not succesfull """ return self._call_rpc( self.scanning_stub.FolderHashScan, @@ -479,7 +479,7 @@ def folder_hash_scan(self, request: Dict) -> Dict: 'Sending folder hash scan data (rqId: {rqId})...', ) - def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional[str] = None) -> dict: + def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional[str] = None) -> Optional[Dict]: """ Call a gRPC method and return the response as a dictionary @@ -490,7 +490,7 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional debug_msg (str, optional): Debug message template that can include {rqId} placeholder. Returns: - dict: The parsed gRPC response as a dictionary, or None if an error occurred. + dict: The parsed gRPC response as a dictionary, or None if something went wrong """ request_id = str(uuid.uuid4()) @@ -510,7 +510,7 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional ) if resp and not self._check_status_response(resp.status, request_id): - raise ScanossGrpcError(f'Unsuccessful status response (rqId: {request_id}).') + return None resp_dict = MessageToDict(resp, preserving_proto_field_name=True) return resp_dict @@ -590,7 +590,7 @@ def get_provenance_json(self, purls: dict) -> dict: return resp_dict return None - def get_provenance_origin(self, request: Dict) -> Dict: + def get_provenance_origin(self, request: Dict) -> Optional[Dict]: """ Client function to call the rpc for GetComponentOrigin @@ -598,7 +598,7 @@ def get_provenance_origin(self, request: Dict) -> Dict: request (Dict): GetComponentOrigin Request Returns: - Dict: OriginResponse + Optional[Dict]: OriginResponse, or None if the request was not successfull """ return self._call_rpc( self.provenance_stub.GetComponentOrigin, From 085f1a5b37458ef726d2371aadf74fd66767e561 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 25 Apr 2025 10:52:44 +0200 Subject: [PATCH 319/489] feat: SP-2388 Update version file --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 324704aa..810cecac 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.22.1' +__version__ = '1.23.0' From f3bdb415f761dd4e031a722bba978d3184694763 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 5 May 2025 11:37:00 +0200 Subject: [PATCH 320/489] Initialize crypto sub command refactor --- src/scanoss/cli.py | 58 +++++++++++++++++++++++++++------------- src/scanoss/constants.py | 2 ++ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 00a97c62..ccdae677 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -50,6 +50,7 @@ from . import __version__ from .components import Components from .constants import ( + DEFAULT_API_TIMEOUT, DEFAULT_POST_SIZE, DEFAULT_RETRY, DEFAULT_TIMEOUT, @@ -292,15 +293,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='component sub-commands', ) - # Component Sub-command: component crypto - c_crypto = comp_sub.add_parser( - 'crypto', - aliases=['cr'], - description=f'Show Cryptographic algorithms: {__version__}', - help='Retrieve cryptographic algorithms for the given components', - ) - c_crypto.set_defaults(func=comp_crypto) - # Component Sub-command: component vulns c_vulns = comp_sub.add_parser( 'vulns', @@ -361,18 +353,48 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_versions.add_argument('--limit', '-l', type=int, help='Generic component search') c_versions.set_defaults(func=comp_versions) + # Sub-command: crypto + p_crypto = subparsers.add_parser( + 'crypto', + aliases=['cr'], + description=f'SCANOSS Crypto commands: {__version__}', + help='Crypto support commands', + ) + crypto_sub = p_crypto.add_subparsers( + title='Crypto Commands', + dest='subparsercmd', + description='crypto sub-commands', + help='crypto sub-commands', + required=True, + ) + + # GetAlgorithms and GetAlgorithmsInRange gRPC APIs + p_crypto_algorithms = crypto_sub.add_parser( + 'algorithms', + aliases=['alg'], + description=f'Show Cryptographic algorithms: {__version__}', + help='Retrieve cryptographic algorithms for the given components', + ) + p_crypto_algorithms.add_argument( + '--range', + '-r', + type=str, + help='Returns the list of versions in the specified range that contains cryptographic algorithms', + ) + p_crypto_algorithms.set_defaults(func=crypto_algorithms) + # Common purl Component sub-command options - for p in [c_crypto, c_vulns, c_semgrep, c_provenance]: + for p in [c_vulns, c_semgrep, c_provenance, p_crypto_algorithms]: p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') # Common Component sub-command options - for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]: + for p in [c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_crypto_algorithms]: p.add_argument( '--timeout', '-M', type=int, - default=600, + default=DEFAULT_API_TIMEOUT, help='Timeout (in seconds) for API communication (optional - default 600)', ) @@ -588,7 +610,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_dep, p_fc, p_cnv, - c_crypto, c_vulns, c_search, c_versions, @@ -597,6 +618,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_c_dwnld, p_folder_scan, p_folder_hash, + p_crypto_algorithms, ]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -674,7 +696,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 # Global Scan/GRPC options for p in [ p_scan, - c_crypto, c_vulns, c_search, c_versions, @@ -682,6 +703,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_provenance, p_folder_scan, p_cs, + p_crypto_algorithms, ]: p.add_argument( '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' @@ -708,7 +730,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) # Global GRPC options - for p in [p_scan, c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan, p_cs]: + for p in [p_scan, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan, p_cs, p_crypto_algorithms]: p.add_argument( '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' ) @@ -751,7 +773,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_c_loc, p_c_dwnld, p_p_proxy, - c_crypto, c_vulns, c_search, c_versions, @@ -763,6 +784,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_folder_scan, p_folder_hash, p_cs, + p_crypto_algorithms, ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') @@ -1393,9 +1415,9 @@ def get_pac_file(pac: str): return pac_file -def comp_crypto(parser, args): +def crypto_algorithms(parser, args): """ - Run the "component crypto" sub-command + Run the "crypto algorithms" sub-command Parameters ---------- parser: ArgumentParser diff --git a/src/scanoss/constants.py b/src/scanoss/constants.py index 3bce0ee9..1dd9bd61 100644 --- a/src/scanoss/constants.py +++ b/src/scanoss/constants.py @@ -10,3 +10,5 @@ DEFAULT_URL = 'https://api.osskb.org' # default free service URL DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL + +DEFAULT_API_TIMEOUT = 600 From c69e9dcc2c484e5dd253efcc6181af70ee9181c9 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 5 May 2025 12:53:58 +0200 Subject: [PATCH 321/489] Move get crypto algorithms to it's own subcommand --- src/scanoss/cli.py | 29 +++--- src/scanoss/cryptography.py | 176 ++++++++++++++++++++++++++++++++++++ src/scanoss/scanossgrpc.py | 65 ++++++------- src/scanoss/utils/file.py | 4 +- 4 files changed, 226 insertions(+), 48 deletions(-) create mode 100644 src/scanoss/cryptography.py diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ccdae677..6b66203f 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -31,6 +31,7 @@ import pypac +from scanoss.cryptography import Cryptography, create_cryptography_config_from_args from scanoss.scanners.container_scanner import ( DEFAULT_SYFT_COMMAND, DEFAULT_SYFT_TIMEOUT, @@ -1432,22 +1433,20 @@ def crypto_algorithms(parser, args): if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') sys.exit(1) - pac_file = get_pac_file(args.pac) - comps = Components( - debug=args.debug, - trace=args.trace, - quiet=args.quiet, - grpc_url=args.api2url, - api_key=args.key, - ca_cert=args.ca_cert, - proxy=args.proxy, - grpc_proxy=args.grpc_proxy, - pac=pac_file, - timeout=args.timeout, - req_headers=process_req_headers(args.header), - ) - if not comps.get_crypto_details(args.input, args.purl, args.output): + try: + config = create_cryptography_config_from_args(args) + grpc_config = create_grpc_config_from_args(args) + client = ScanossGrpc(**asdict(grpc_config)) + + # TODO: Add PAC file support + # pac_file = get_pac_file(config.pac) + + cryptography = Cryptography(config=config, client=client) + cryptography.get_algorithms() + cryptography.present(output_file=args.output) + except Exception as e: + print_stderr(f'ERROR: {e}') sys.exit(1) diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py new file mode 100644 index 00000000..2f1c9878 --- /dev/null +++ b/src/scanoss/cryptography.py @@ -0,0 +1,176 @@ +import json +from dataclasses import dataclass +from typing import Dict, Optional + +from scanoss.scanossbase import ScanossBase +from scanoss.scanossgrpc import ScanossGrpc +from scanoss.utils.abstract_presenter import AbstractPresenter +from scanoss.utils.file import validate_json_file + + +class ScanossCryptographyError(Exception): + pass + + +@dataclass +class CryptographyConfig: + debug: bool = False + trace: bool = False + quiet: bool = False + get_range: bool = False + purl: str = None + input_file: str = None + output_file: str = None + header: str = None + + +def create_cryptography_config_from_args(args) -> CryptographyConfig: + return CryptographyConfig( + debug=getattr(args, 'debug', None), + trace=getattr(args, 'trace', None), + quiet=getattr(args, 'quiet', None), + get_range=getattr(args, 'range', None), + purl=getattr(args, 'purl', None), + input_file=getattr(args, 'input', None), + output_file=getattr(args, 'output', None), + header=getattr(args, 'header', None), + ) + + +class Cryptography: + """ + Cryptography Class + + This class is used to decorate purls with cryptography information. + """ + + def __init__( + self, + config: CryptographyConfig, + client: ScanossGrpc, + ): + """ + Initialize the Cryptography. + + Args: + config (CryptographyConfig): Configuration parameters for the cryptography. + client (ScanossGrpc): gRPC client for communicating with the scanning service. + """ + self.base = ScanossBase( + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + self.presenter = CryptographyPresenter( + self, + debug=config.debug, + trace=config.trace, + quiet=config.quiet, + ) + + self.client = client + self.config = config + self.purls_request = self._build_purls_request() + self.results = None + + def get_algorithms(self) -> Optional[Dict]: + """ + Get the cryptographic algorithms for the provided purl or input file. + + Returns: + Optional[Dict]: The folder hash response from the gRPC client, or None if an error occurs. + """ + + try: + self.base.print_stderr( + f'Getting cryptographic algorithms for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' + ) + if self.config.get_range: + response = self.client.get_crypto_algorithms_in_range_for_purl(self.purls_request) + else: + response = self.client.get_crypto_algorithms_for_purl(self.purls_request) + if response: + self.results = response + except Exception as e: + raise ScanossCryptographyError(f'Problem with purl input file. {e}') + + return self.results + + def _build_purls_request( + self, + ) -> Optional[dict]: + """ + Load the specified purls from a JSON file or a list of PURLs and return a dictionary + + Args: + json_file (Optional[str], optional): The JSON file containing the PURLs. Defaults to None. + purls (Optional[List[str]], optional): The list of PURLs. Defaults to None. + + Returns: + Optional[dict]: The dictionary containing the PURLs + """ + if self.config.input_file: + input_file_validation = validate_json_file(self.config.input_file) + if not input_file_validation.is_valid: + self.base.print_stderr( + f'ERROR: The supplied input file "{self.config.input_file}" was not found or is empty.' + ) + raise Exception(f'Problem with purl input file. {input_file_validation.error}') + # Validate the input file is in PurlRequest format + if ( + not isinstance(input_file_validation.data, dict) + or 'purls' not in input_file_validation.data + or not isinstance(input_file_validation.data['purls'], list) + or not all(isinstance(p, dict) and 'purl' in p for p in input_file_validation.data['purls']) + ): + raise Exception('The supplied input file is not in the correct PurlRequest format.') + return input_file_validation.data + if self.config.purl: + return {'purls': [{'purl': p} for p in self.config.purl]} + return None + + def present(self, output_format: str = None, output_file: str = None): + """Present the results in the selected format""" + self.presenter.present(output_format=output_format, output_file=output_file) + + +class CryptographyPresenter(AbstractPresenter): + """ + Cryptography presenter class + Handles the presentation of the cryptography results + """ + + def __init__(self, cryptography: Cryptography, **kwargs): + super().__init__(**kwargs) + self.cryptography = cryptography + + def _format_json_output(self) -> str: + """ + Format the scan output data into a JSON object + + Returns: + str: The formatted JSON string + """ + return json.dumps(self.cryptography.results, indent=2) + + def _format_plain_output(self) -> str: + """ + Format the scan output data into a plain text string + """ + return ( + json.dumps(self.cryptography.results, indent=2) + if isinstance(self.cryptography.results, dict) + else str(self.cryptography.results) + ) + + def _format_cyclonedx_output(self) -> str: + raise NotImplementedError('CycloneDX output is not implemented') + + def _format_spdxlite_output(self) -> str: + raise NotImplementedError('SPDXlite output is not implemented') + + def _format_csv_output(self) -> str: + raise NotImplementedError('CSV output is not implemented') + + def _format_raw_output(self) -> str: + raise NotImplementedError('Raw output is not implemented') diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 17752f4d..c840d15f 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -54,7 +54,6 @@ CompVersionResponse, ) from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub -from .api.cryptography.v2.scanoss_cryptography_pb2 import AlgorithmResponse from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub @@ -312,36 +311,6 @@ def process_file(file): merged_response['status'] = response['status'] return merged_response - def get_crypto_json(self, purls: dict) -> dict: - """ - Client function to call the rpc for Cryptography GetAlgorithms - :param purls: Message to send to the service - :return: Server response or None - """ - if not purls: - self.print_stderr('ERROR: No message supplied to send to gRPC service.') - return None - request_id = str(uuid.uuid4()) - resp: AlgorithmResponse - try: - request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...') - resp = self.crypto_stub.GetAlgorithms(request, metadata=metadata, timeout=self.timeout) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - if not self._check_status_response(resp.status, request_id): - return None - resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict - del resp_dict['status'] - return resp_dict - return None - def get_vulnerabilities_json(self, purls: dict) -> dict: """ Client function to call the rpc for Vulnerability GetVulnerabilities @@ -607,6 +576,40 @@ def get_provenance_origin(self, request: Dict) -> Optional[Dict]: 'Sending data for provenance origin decoration (rqId: {rqId})...', ) + def get_crypto_algorithms_for_purl(self, request: Dict) -> Optional[Dict]: + """ + Client function to call the rpc for GetAlgorithms for a list of purls + + Args: + request (Dict): PurlRequest + + Returns: + Optional[Dict]: AlgorithmResponse, or None if the request was not successfull + """ + return self._call_rpc( + self.crypto_stub.GetAlgorithms, + request, + PurlRequest, + 'Sending data for cryptographic algorithms decoration (rqId: {rqId})...', + ) + + def get_crypto_algorithms_in_range_for_purl(self, request: Dict) -> Optional[Dict]: + """ + Client function to call the rpc for GetAlgorithmsInRange for a list of purls + + Args: + request (Dict): PurlRequest + + Returns: + Optional[Dict]: AlgorithmsInRangeResponse, or None if the request was not successfull + """ + return self._call_rpc( + self.crypto_stub.GetAlgorithmsInRange, + request, + PurlRequest, + 'Sending data for cryptographic algorithms in range decoration (rqId: {rqId})...', + ) + def load_generic_headers(self): """ Adds custom headers from req_headers to metadata. diff --git a/src/scanoss/utils/file.py b/src/scanoss/utils/file.py index ab28327c..84c6dc47 100644 --- a/src/scanoss/utils/file.py +++ b/src/scanoss/utils/file.py @@ -49,8 +49,8 @@ def validate_json_file(json_file_path: str) -> JsonValidation: json_file_path (str): The JSON file to validate Returns: - Tuple[bool, str]: A tuple containing a boolean indicating if the file is valid and a message - """ + JsonValidation: A JsonValidation object containing a boolean indicating if the file is valid, the data, error, and error code + """ # noqa: E501 if not json_file_path: return JsonValidation(is_valid=False, error='No JSON file specified') if not os.path.isfile(json_file_path): From 75bc9bf52b054714e779cd2ead03f432cbec2746 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 5 May 2025 13:29:23 +0200 Subject: [PATCH 322/489] Address pr comments --- src/scanoss/cli.py | 7 +++++++ src/scanoss/cryptography.py | 33 +++++++++++++++------------------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 6b66203f..71370e8d 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1445,7 +1445,14 @@ def crypto_algorithms(parser, args): cryptography = Cryptography(config=config, client=client) cryptography.get_algorithms() cryptography.present(output_file=args.output) + except ScanossGrpcError as e: + print_stderr(f'API ERROR: {e}') + sys.exit(1) except Exception as e: + if args.debug: + import traceback + + traceback.print_exc() print_stderr(f'ERROR: {e}') sys.exit(1) diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index 2f1c9878..83ea1800 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -1,6 +1,6 @@ import json from dataclasses import dataclass -from typing import Dict, Optional +from typing import Dict, List, Optional from scanoss.scanossbase import ScanossBase from scanoss.scanossgrpc import ScanossGrpc @@ -18,7 +18,7 @@ class CryptographyConfig: trace: bool = False quiet: bool = False get_range: bool = False - purl: str = None + purl: List[str] = None input_file: str = None output_file: str = None header: str = None @@ -81,18 +81,17 @@ def get_algorithms(self) -> Optional[Dict]: Optional[Dict]: The folder hash response from the gRPC client, or None if an error occurs. """ - try: - self.base.print_stderr( - f'Getting cryptographic algorithms for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' - ) - if self.config.get_range: - response = self.client.get_crypto_algorithms_in_range_for_purl(self.purls_request) - else: - response = self.client.get_crypto_algorithms_for_purl(self.purls_request) - if response: - self.results = response - except Exception as e: - raise ScanossCryptographyError(f'Problem with purl input file. {e}') + if not self.purls_request: + raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') + self.base.print_stderr( + f'Getting cryptographic algorithms for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' + ) + if self.config.get_range: + response = self.client.get_crypto_algorithms_in_range_for_purl(self.purls_request) + else: + response = self.client.get_crypto_algorithms_for_purl(self.purls_request) + if response: + self.results = response return self.results @@ -112,10 +111,8 @@ def _build_purls_request( if self.config.input_file: input_file_validation = validate_json_file(self.config.input_file) if not input_file_validation.is_valid: - self.base.print_stderr( - f'ERROR: The supplied input file "{self.config.input_file}" was not found or is empty.' - ) - raise Exception(f'Problem with purl input file. {input_file_validation.error}') + raise Exception(f'There was a problem with the purl input file. {input_file_validation.error}') + # Validate the input file is in PurlRequest format if ( not isinstance(input_file_validation.data, dict) From 84ea3a40f8b868dc4d450f263c576193ebb510b4 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 5 May 2025 15:54:20 +0200 Subject: [PATCH 323/489] Add get crypto hints and versions in range sub-commands --- src/scanoss/cli.py | 141 ++++++++++++++++++++++++++++++++++-- src/scanoss/cryptography.py | 51 ++++++++++++- src/scanoss/scanossgrpc.py | 51 +++++++++++++ 3 files changed, 234 insertions(+), 9 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 71370e8d..773e8889 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -377,20 +377,49 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Retrieve cryptographic algorithms for the given components', ) p_crypto_algorithms.add_argument( - '--range', - '-r', - type=str, + '--with-range', + action='store_true', help='Returns the list of versions in the specified range that contains cryptographic algorithms', ) p_crypto_algorithms.set_defaults(func=crypto_algorithms) + # GetEncryptionHints and GetHintsInRange gRPC APIs + p_crypto_hints = crypto_sub.add_parser( + 'hints', + description=f'Show Encryption hints: {__version__}', + help='Retrieve encryption hints for the given components', + ) + p_crypto_hints.add_argument( + '--with-range', + action='store_true', + help='Returns the list of versions in the specified range that contains encryption hints', + ) + p_crypto_hints.set_defaults(func=crypto_hints) + + p_crypto_versions_in_range = crypto_sub.add_parser( + 'versions-in-range', + aliases=['vr'], + description=f'Show versions in range: {__version__}', + help="Given a list of PURLS and version ranges, get a list of versions that do/don't contain crypto algorithms", + ) + p_crypto_versions_in_range.set_defaults(func=crypto_versions_in_range) + # Common purl Component sub-command options - for p in [c_vulns, c_semgrep, c_provenance, p_crypto_algorithms]: + for p in [c_vulns, c_semgrep, c_provenance, p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range]: p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') # Common Component sub-command options - for p in [c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_crypto_algorithms]: + for p in [ + c_vulns, + c_search, + c_versions, + c_semgrep, + c_provenance, + p_crypto_algorithms, + p_crypto_hints, + p_crypto_versions_in_range, + ]: p.add_argument( '--timeout', '-M', @@ -620,6 +649,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_folder_scan, p_folder_hash, p_crypto_algorithms, + p_crypto_hints, + p_crypto_versions_in_range, ]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -705,6 +736,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_folder_scan, p_cs, p_crypto_algorithms, + p_crypto_hints, + p_crypto_versions_in_range, ]: p.add_argument( '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' @@ -731,7 +764,19 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) # Global GRPC options - for p in [p_scan, c_vulns, c_search, c_versions, c_semgrep, c_provenance, p_folder_scan, p_cs, p_crypto_algorithms]: + for p in [ + p_scan, + c_vulns, + c_search, + c_versions, + c_semgrep, + c_provenance, + p_folder_scan, + p_cs, + p_crypto_algorithms, + p_crypto_hints, + p_crypto_versions_in_range, + ]: p.add_argument( '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' ) @@ -786,6 +831,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_folder_hash, p_cs, p_crypto_algorithms, + p_crypto_hints, + p_crypto_versions_in_range, ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') @@ -1457,6 +1504,88 @@ def crypto_algorithms(parser, args): sys.exit(1) +def crypto_hints(parser, args): + """ + Run the "crypto hints" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + sys.exit(1) + + try: + config = create_cryptography_config_from_args(args) + grpc_config = create_grpc_config_from_args(args) + client = ScanossGrpc(**asdict(grpc_config)) + + # TODO: Add PAC file support + # pac_file = get_pac_file(config.pac) + + cryptography = Cryptography(config=config, client=client) + cryptography.get_encryption_hints() + cryptography.present(output_file=args.output) + except ScanossGrpcError as e: + print_stderr(f'API ERROR: {e}') + sys.exit(1) + except Exception as e: + if args.debug: + import traceback + + traceback.print_exc() + print_stderr(f'ERROR: {e}') + sys.exit(1) + + +def crypto_versions_in_range(parser, args): + """ + Run the "crypto versions-in-range" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') + sys.exit(1) + + try: + config = create_cryptography_config_from_args(args) + grpc_config = create_grpc_config_from_args(args) + client = ScanossGrpc(**asdict(grpc_config)) + + # TODO: Add PAC file support + # pac_file = get_pac_file(config.pac) + + cryptography = Cryptography(config=config, client=client) + cryptography.get_versions_in_range() + cryptography.present(output_file=args.output) + except ScanossGrpcError as e: + print_stderr(f'API ERROR: {e}') + sys.exit(1) + except Exception as e: + if args.debug: + import traceback + + traceback.print_exc() + print_stderr(f'ERROR: {e}') + sys.exit(1) + + def comp_vulns(parser, args): """ Run the "component vulns" sub-command diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index 83ea1800..76379f24 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -17,7 +17,7 @@ class CryptographyConfig: debug: bool = False trace: bool = False quiet: bool = False - get_range: bool = False + with_range: bool = False purl: List[str] = None input_file: str = None output_file: str = None @@ -29,7 +29,7 @@ def create_cryptography_config_from_args(args) -> CryptographyConfig: debug=getattr(args, 'debug', None), trace=getattr(args, 'trace', None), quiet=getattr(args, 'quiet', None), - get_range=getattr(args, 'range', None), + with_range=getattr(args, 'with_range', None), purl=getattr(args, 'purl', None), input_file=getattr(args, 'input', None), output_file=getattr(args, 'output', None), @@ -86,7 +86,7 @@ def get_algorithms(self) -> Optional[Dict]: self.base.print_stderr( f'Getting cryptographic algorithms for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' ) - if self.config.get_range: + if self.config.with_range: response = self.client.get_crypto_algorithms_in_range_for_purl(self.purls_request) else: response = self.client.get_crypto_algorithms_for_purl(self.purls_request) @@ -95,6 +95,51 @@ def get_algorithms(self) -> Optional[Dict]: return self.results + def get_encryption_hints(self) -> Optional[Dict]: + """ + Get the encryption hints for the provided purl or input file. + + Returns: + Optional[Dict]: The encryption hints response from the gRPC client, or None if an error occurs. + """ + + if not self.purls_request: + raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') + self.base.print_stderr( + f'Getting encryption hints ' + f'{"in range" if self.config.with_range else ""} ' + f'for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' + ) + if self.config.with_range: + response = self.client.get_encryption_hints_in_range_for_purl(self.purls_request) + else: + response = self.client.get_encryption_hints_for_purl(self.purls_request) + if response: + self.results = response + + return self.results + + def get_versions_in_range(self) -> Optional[Dict]: + """ + Given a list of PURLS and version ranges, get a list of versions that do/do not contain cryptographic algorithms + + Returns: + Optional[Dict]: The versions in range response from the gRPC client, or None if an error occurs. + """ + + if not self.purls_request: + raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') + + self.base.print_stderr( + f'Getting versions in range for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' + ) + + response = self.client.get_versions_in_range_for_purl(self.purls_request) + if response: + self.results = response + + return self.results + def _build_purls_request( self, ) -> Optional[dict]: diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index c840d15f..8122ae51 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -610,6 +610,57 @@ def get_crypto_algorithms_in_range_for_purl(self, request: Dict) -> Optional[Dic 'Sending data for cryptographic algorithms in range decoration (rqId: {rqId})...', ) + def get_encryption_hints_for_purl(self, request: Dict) -> Optional[Dict]: + """ + Client function to call the rpc for GetEncryptionHints for a list of purls + + Args: + request (Dict): PurlRequest + + Returns: + Optional[Dict]: HintsResponse, or None if the request was not successfull + """ + return self._call_rpc( + self.crypto_stub.GetEncryptionHints, + request, + PurlRequest, + 'Sending data for encryption hints decoration (rqId: {rqId})...', + ) + + def get_encryption_hints_in_range_for_purl(self, request: Dict) -> Optional[Dict]: + """ + Client function to call the rpc for GetHintsInRange for a list of purls + + Args: + request (Dict): PurlRequest + + Returns: + Optional[Dict]: HintsInRangeResponse, or None if the request was not successfull + """ + return self._call_rpc( + self.crypto_stub.GetHintsInRange, + request, + PurlRequest, + 'Sending data for encryption hints in range decoration (rqId: {rqId})...', + ) + + def get_versions_in_range_for_purl(self, request: Dict) -> Optional[Dict]: + """ + Client function to call the rpc for GetVersionsInRange for a list of purls + + Args: + request (Dict): PurlRequest + + Returns: + Optional[Dict]: VersionsInRangeResponse, or None if the request was not successfull + """ + return self._call_rpc( + self.crypto_stub.GetVersionsInRange, + request, + PurlRequest, + 'Sending data for cryptographic versions in range decoration (rqId: {rqId})...', + ) + def load_generic_headers(self): """ Adds custom headers from req_headers to metadata. From 2133ef8eaee0e67322586b9300294c9abd8dc227 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 6 May 2025 14:56:47 +0200 Subject: [PATCH 324/489] Update changelog and docs --- CHANGELOG.md | 9 ++++- docs/source/index.rst | 85 +++++++++++++++++++++++++++++++++++++++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 5 ++- 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40c997f2..d8a24a64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.24.0] - 2025-05-06 +### Added +- Add `crypto` subcommand to retrieve cryptographic algorithms for the given components +- Add `crypto hints` subcommand to retrieve cryptographic hints for the given components +- Add `crypto versions-in-range` subcommand to retrieve cryptographic versions in range for the given components + ## [1.23.0] - 2025-04-24 ### Added - Add `--origin` flag to `component provenance` subcommand to retrieve provenance using contributors origin @@ -515,4 +521,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.20.6]: https://github.com/scanoss/scanoss.py/compare/v1.20.5...v1.20.6 [1.21.0]: https://github.com/scanoss/scanoss.py/compare/v1.20.6...v1.21.0 [1.22.0]: https://github.com/scanoss/scanoss.py/compare/v1.21.0...v1.22.0 -[1.23.0]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.23.0 \ No newline at end of file +[1.23.0]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.23.0 +[1.24.0]: https://github.com/scanoss/scanoss.py/compare/v1.23.0...v1.24.0 \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index ba3e8d83..f70f93d7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -336,6 +336,91 @@ Scans Docker container images for dependencies, extracting and analyzing compone * - --ca-cert - Alternative certificate PEM file +----------------- +Crypto: crypto, cr +----------------- + +Provides subcommands to retrieve cryptographic information for components. + +.. code-block:: bash + + scanoss-py crypto + +Subcommands: +~~~~~~~~~~~~ + +**algorithms (alg)** + Retrieve cryptographic algorithms for the given components. + + .. code-block:: bash + + scanoss-py crypto algorithms --purl + + .. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --with-range + - Returns the list of versions in the specified range that contains cryptographic algorithms. (Replaces the previous --range option) + +**hints** + Retrieve encryption hints for the given components. + + .. code-block:: bash + + scanoss-py crypto hints --purl + + .. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --with-range + - Returns the list of versions in the specified range that contains encryption hints. + +**versions-in-range (vr)** + Given a list of PURLs and version ranges, get a list of versions that do/don't contain crypto algorithms. + + .. code-block:: bash + + scanoss-py crypto versions-in-range --purl + +Common Crypto Arguments: +~~~~~~~~~~~~~~~~~~~~~~~~ +The following arguments are common to the ``algorithms``, ``hints``, and ``versions-in-range`` subcommands: + +.. list-table:: + :widths: 20 30 + :header-rows: 1 + + * - Argument + - Description + * - --purl , -p + - Package URL (PURL) to process. Can be specified multiple times. + * - --input , -i + - Input file name containing PURLs. + * - --output , -o + - Output result file name (optional - default STDOUT). + * - --timeout , -M + - Timeout (in seconds) for API communication (optional - default 600). + * - --key , -k + - SCANOSS API Key token (optional - not required for default OSSKB URL). + * - --api2url + - SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org). + * - --grpc-proxy + - GRPC Proxy URL to use for connections. + * - --ca-cert + - Alternative certificate PEM file. + * - --debug, -d + - Enable debug messages. + * - --trace, -t + - Enable trace messages, including API posts. + * - --quiet, -q + - Enable quiet mode. + ----------------- Component: ----------------- diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 810cecac..228a61a1 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.23.0' +__version__ = '1.24.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 773e8889..84dcefea 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -366,7 +366,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 dest='subparsercmd', description='crypto sub-commands', help='crypto sub-commands', - required=True, ) # GetAlgorithms and GetAlgorithmsInRange gRPC APIs @@ -845,7 +844,9 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 if not args.subparser: parser.print_help() # No sub command subcommand, print general help sys.exit(1) - elif (args.subparser in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins')) and not args.subparsercmd: + elif ( + args.subparser in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins', 'crypto', 'cr') + ) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed sys.exit(1) args.func(parser, args) # Execute the function associated with the sub-command From 3a2dff6b09b22880b074cc33527cb8997a2f83e8 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 6 May 2025 15:08:14 +0200 Subject: [PATCH 325/489] Add pac file and headers support --- src/scanoss/cli.py | 21 ++++++++++++--------- src/scanoss/scanossgrpc.py | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 84dcefea..ec88177a 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1485,11 +1485,12 @@ def crypto_algorithms(parser, args): try: config = create_cryptography_config_from_args(args) grpc_config = create_grpc_config_from_args(args) + if args.pac: + grpc_config.pac = get_pac_file(args.pac) + if args.header: + grpc_config.req_headers = process_req_headers(args.header) client = ScanossGrpc(**asdict(grpc_config)) - # TODO: Add PAC file support - # pac_file = get_pac_file(config.pac) - cryptography = Cryptography(config=config, client=client) cryptography.get_algorithms() cryptography.present(output_file=args.output) @@ -1526,11 +1527,12 @@ def crypto_hints(parser, args): try: config = create_cryptography_config_from_args(args) grpc_config = create_grpc_config_from_args(args) + if args.pac: + grpc_config.pac = get_pac_file(args.pac) + if args.header: + grpc_config.req_headers = process_req_headers(args.header) client = ScanossGrpc(**asdict(grpc_config)) - # TODO: Add PAC file support - # pac_file = get_pac_file(config.pac) - cryptography = Cryptography(config=config, client=client) cryptography.get_encryption_hints() cryptography.present(output_file=args.output) @@ -1567,11 +1569,12 @@ def crypto_versions_in_range(parser, args): try: config = create_cryptography_config_from_args(args) grpc_config = create_grpc_config_from_args(args) + if args.pac: + grpc_config.pac = get_pac_file(args.pac) + if args.header: + grpc_config.req_headers = process_req_headers(args.header) client = ScanossGrpc(**asdict(grpc_config)) - # TODO: Add PAC file support - # pac_file = get_pac_file(config.pac) - cryptography = Cryptography(config=config, client=client) cryptography.get_versions_in_range() cryptography.present(output_file=args.output) diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 8122ae51..189f4c10 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -691,10 +691,11 @@ class GrpcConfig: quiet: Optional[bool] = False ver_details: Optional[str] = None ca_cert: Optional[str] = None - pac: Optional[PACFile] = None timeout: Optional[int] = DEFAULT_TIMEOUT proxy: Optional[str] = None grpc_proxy: Optional[str] = None + pac: Optional[PACFile] = None + req_headers: Optional[dict] = None def create_grpc_config_from_args(args) -> GrpcConfig: @@ -706,7 +707,6 @@ def create_grpc_config_from_args(args) -> GrpcConfig: quiet=getattr(args, 'quiet', False), ver_details=getattr(args, 'ver_details', None), ca_cert=getattr(args, 'ca_cert', None), - pac=getattr(args, 'pac', None), timeout=getattr(args, 'timeout', DEFAULT_TIMEOUT), proxy=getattr(args, 'proxy', None), grpc_proxy=getattr(args, 'grpc_proxy', None), From 0fc961b48cd5524cba89cac2b69fd17ca951b1cf Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 9 May 2025 16:47:11 +0200 Subject: [PATCH 326/489] Validate purl inputs --- src/scanoss/cryptography.py | 139 ++++++++++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 29 deletions(-) diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index 76379f24..c504c7b8 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -14,23 +14,62 @@ class ScanossCryptographyError(Exception): @dataclass class CryptographyConfig: + purl: List[str] + input_file: Optional[str] = None + output_file: Optional[str] = None + header: Optional[str] = None debug: bool = False trace: bool = False quiet: bool = False with_range: bool = False - purl: List[str] = None - input_file: str = None - output_file: str = None - header: str = None + + def __post_init__(self): + # If with_range is True, purls must contain "@" + if self.with_range: + if self.purl: + for purl in self.purl: + parts = purl.split('@') + if not (len(parts) >= 2 and parts[1]): + raise ValueError( + f'Invalid PURL format: "{purl}".' + f'It must include a version (e.g., pkg:type/name@version)' + ) + if self.input_file: + input_file_validation = validate_json_file(self.input_file) + if not input_file_validation.is_valid: + raise Exception( + f'There was a problem with the purl input file. {input_file_validation.error}' + ) + + # Validate the input file is in PurlRequest format + if ( + not isinstance(input_file_validation.data, dict) + or 'purls' not in input_file_validation.data + or not isinstance(input_file_validation.data['purls'], list) + or not all( + isinstance(p, dict) and 'purl' in p + for p in input_file_validation.data['purls'] + ) + ): + raise ValueError( + 'The supplied input file is not in the correct PurlRequest format.' + ) + if self.with_range: + purls = input_file_validation.data['purls'] + if any('requirement' not in p for p in purls): + raise ValueError( + 'One or more PURLs are missing the "requirement" field.' + ) + return input_file_validation.data def create_cryptography_config_from_args(args) -> CryptographyConfig: return CryptographyConfig( - debug=getattr(args, 'debug', None), - trace=getattr(args, 'trace', None), - quiet=getattr(args, 'quiet', None), - with_range=getattr(args, 'with_range', None), - purl=getattr(args, 'purl', None), + debug=getattr(args, 'debug', False), + trace=getattr(args, 'trace', False), + quiet=getattr(args, 'quiet', False), + with_range=getattr(args, 'with_range', False), + purl=getattr(args, 'purl', []), input_file=getattr(args, 'input', None), output_file=getattr(args, 'output', None), header=getattr(args, 'header', None), @@ -82,14 +121,20 @@ def get_algorithms(self) -> Optional[Dict]: """ if not self.purls_request: - raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') + raise ScanossCryptographyError( + 'No PURLs supplied. Provide --purl or --input.' + ) self.base.print_stderr( f'Getting cryptographic algorithms for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' ) if self.config.with_range: - response = self.client.get_crypto_algorithms_in_range_for_purl(self.purls_request) + response = self.client.get_crypto_algorithms_in_range_for_purl( + self.purls_request + ) else: - response = self.client.get_crypto_algorithms_for_purl(self.purls_request) + response = self.client.get_crypto_algorithms_for_purl( + self.purls_request + ) if response: self.results = response @@ -104,16 +149,22 @@ def get_encryption_hints(self) -> Optional[Dict]: """ if not self.purls_request: - raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') + raise ScanossCryptographyError( + 'No PURLs supplied. Provide --purl or --input.' + ) self.base.print_stderr( f'Getting encryption hints ' f'{"in range" if self.config.with_range else ""} ' f'for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' ) if self.config.with_range: - response = self.client.get_encryption_hints_in_range_for_purl(self.purls_request) + response = self.client.get_encryption_hints_in_range_for_purl( + self.purls_request + ) else: - response = self.client.get_encryption_hints_for_purl(self.purls_request) + response = self.client.get_encryption_hints_for_purl( + self.purls_request + ) if response: self.results = response @@ -128,13 +179,17 @@ def get_versions_in_range(self) -> Optional[Dict]: """ if not self.purls_request: - raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') + raise ScanossCryptographyError( + 'No PURLs supplied. Provide --purl or --input.' + ) self.base.print_stderr( f'Getting versions in range for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' ) - response = self.client.get_versions_in_range_for_purl(self.purls_request) + response = self.client.get_versions_in_range_for_purl( + self.purls_request + ) if response: self.results = response @@ -156,24 +211,50 @@ def _build_purls_request( if self.config.input_file: input_file_validation = validate_json_file(self.config.input_file) if not input_file_validation.is_valid: - raise Exception(f'There was a problem with the purl input file. {input_file_validation.error}') + raise Exception( + f'There was a problem with the purl input file. {input_file_validation.error}' + ) - # Validate the input file is in PurlRequest format - if ( - not isinstance(input_file_validation.data, dict) - or 'purls' not in input_file_validation.data - or not isinstance(input_file_validation.data['purls'], list) - or not all(isinstance(p, dict) and 'purl' in p for p in input_file_validation.data['purls']) - ): - raise Exception('The supplied input file is not in the correct PurlRequest format.') return input_file_validation.data if self.config.purl: - return {'purls': [{'purl': p} for p in self.config.purl]} + return { + 'purls': [ + { + 'purl': p, + 'requirement': self._extract_version_from_purl(p), + } + for p in self.config.purl + ] + } return None - def present(self, output_format: str = None, output_file: str = None): + def _extract_version_from_purl(self, purl: str) -> str: + """ + Extract version from purl + + Args: + purl (str): The purl string to extract the version from + + Returns: + str: The extracted version + + Raises: + ValueError: If the purl is not in the correct format + """ + try: + return purl.split('@')[-1] + except IndexError: + raise ValueError(f'Invalid purl format: {purl}') + + def present( + self, + output_format: Optional[str] = None, + output_file: Optional[str] = None, + ): """Present the results in the selected format""" - self.presenter.present(output_format=output_format, output_file=output_file) + self.presenter.present( + output_format=output_format, output_file=output_file + ) class CryptographyPresenter(AbstractPresenter): From 8e18dd092de4692eb622b3e98a687ca8a54b7024 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 9 May 2025 17:21:07 +0200 Subject: [PATCH 327/489] Fix lint issue --- src/scanoss/cryptography.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index c504c7b8..9a774134 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -12,6 +12,9 @@ class ScanossCryptographyError(Exception): pass +MIN_SPLIT_PARTS = 2 + + @dataclass class CryptographyConfig: purl: List[str] @@ -29,7 +32,7 @@ def __post_init__(self): if self.purl: for purl in self.purl: parts = purl.split('@') - if not (len(parts) >= 2 and parts[1]): + if not (len(parts) >= MIN_SPLIT_PARTS and parts[1]): raise ValueError( f'Invalid PURL format: "{purl}".' f'It must include a version (e.g., pkg:type/name@version)' From b2d82dddfacdc6fa0d131856a409fdc9ad9e54a9 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 9 May 2025 18:04:30 +0200 Subject: [PATCH 328/489] Validation enhancements --- src/scanoss/cryptography.py | 108 +++++++++++++----------------------- 1 file changed, 40 insertions(+), 68 deletions(-) diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index 9a774134..7fcc5989 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -27,43 +27,43 @@ class CryptographyConfig: with_range: bool = False def __post_init__(self): - # If with_range is True, purls must contain "@" - if self.with_range: - if self.purl: + """ + Validate that the configuration is valid. + """ + if self.purl: + if self.with_range: for purl in self.purl: parts = purl.split('@') if not (len(parts) >= MIN_SPLIT_PARTS and parts[1]): - raise ValueError( - f'Invalid PURL format: "{purl}".' - f'It must include a version (e.g., pkg:type/name@version)' + raise ScanossCryptographyError( + f'Invalid PURL format: "{purl}".' f'It must include a version (e.g., pkg:type/name@version)' ) if self.input_file: input_file_validation = validate_json_file(self.input_file) if not input_file_validation.is_valid: - raise Exception( + raise ScanossCryptographyError( f'There was a problem with the purl input file. {input_file_validation.error}' ) - - # Validate the input file is in PurlRequest format if ( not isinstance(input_file_validation.data, dict) or 'purls' not in input_file_validation.data or not isinstance(input_file_validation.data['purls'], list) - or not all( - isinstance(p, dict) and 'purl' in p - for p in input_file_validation.data['purls'] - ) + or not all(isinstance(p, dict) and 'purl' in p for p in input_file_validation.data['purls']) ): - raise ValueError( - 'The supplied input file is not in the correct PurlRequest format.' - ) + raise ScanossCryptographyError('The supplied input file is not in the correct PurlRequest format.') + purls = input_file_validation.data['purls'] + purls_with_requirement = [] if self.with_range: - purls = input_file_validation.data['purls'] if any('requirement' not in p for p in purls): - raise ValueError( - 'One or more PURLs are missing the "requirement" field.' + raise ScanossCryptographyError( + f'One or more PURLs in "{self.input_file}" are missing the "requirement" field.' ) - return input_file_validation.data + else: + for purl in purls: + purls_with_requirement.append(f'{purl["purl"]}@{purl["requirement"]}') + else: + purls_with_requirement = purls + self.purl = purls_with_requirement def create_cryptography_config_from_args(args) -> CryptographyConfig: @@ -124,20 +124,14 @@ def get_algorithms(self) -> Optional[Dict]: """ if not self.purls_request: - raise ScanossCryptographyError( - 'No PURLs supplied. Provide --purl or --input.' - ) + raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') self.base.print_stderr( f'Getting cryptographic algorithms for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' ) if self.config.with_range: - response = self.client.get_crypto_algorithms_in_range_for_purl( - self.purls_request - ) + response = self.client.get_crypto_algorithms_in_range_for_purl(self.purls_request) else: - response = self.client.get_crypto_algorithms_for_purl( - self.purls_request - ) + response = self.client.get_crypto_algorithms_for_purl(self.purls_request) if response: self.results = response @@ -152,22 +146,16 @@ def get_encryption_hints(self) -> Optional[Dict]: """ if not self.purls_request: - raise ScanossCryptographyError( - 'No PURLs supplied. Provide --purl or --input.' - ) + raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') self.base.print_stderr( f'Getting encryption hints ' f'{"in range" if self.config.with_range else ""} ' f'for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' ) if self.config.with_range: - response = self.client.get_encryption_hints_in_range_for_purl( - self.purls_request - ) + response = self.client.get_encryption_hints_in_range_for_purl(self.purls_request) else: - response = self.client.get_encryption_hints_for_purl( - self.purls_request - ) + response = self.client.get_encryption_hints_for_purl(self.purls_request) if response: self.results = response @@ -182,17 +170,13 @@ def get_versions_in_range(self) -> Optional[Dict]: """ if not self.purls_request: - raise ScanossCryptographyError( - 'No PURLs supplied. Provide --purl or --input.' - ) + raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') self.base.print_stderr( f'Getting versions in range for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' ) - response = self.client.get_versions_in_range_for_purl( - self.purls_request - ) + response = self.client.get_versions_in_range_for_purl(self.purls_request) if response: self.results = response @@ -211,25 +195,15 @@ def _build_purls_request( Returns: Optional[dict]: The dictionary containing the PURLs """ - if self.config.input_file: - input_file_validation = validate_json_file(self.config.input_file) - if not input_file_validation.is_valid: - raise Exception( - f'There was a problem with the purl input file. {input_file_validation.error}' - ) - - return input_file_validation.data - if self.config.purl: - return { - 'purls': [ - { - 'purl': p, - 'requirement': self._extract_version_from_purl(p), - } - for p in self.config.purl - ] - } - return None + return { + 'purls': [ + { + 'purl': p, + 'requirement': self._extract_version_from_purl(p), + } + for p in self.config.purl + ] + } def _extract_version_from_purl(self, purl: str) -> str: """ @@ -242,12 +216,12 @@ def _extract_version_from_purl(self, purl: str) -> str: str: The extracted version Raises: - ValueError: If the purl is not in the correct format + ScanossCryptographyError: If the purl is not in the correct format """ try: return purl.split('@')[-1] except IndexError: - raise ValueError(f'Invalid purl format: {purl}') + raise ScanossCryptographyError(f'Invalid purl format: {purl}') def present( self, @@ -255,9 +229,7 @@ def present( output_file: Optional[str] = None, ): """Present the results in the selected format""" - self.presenter.present( - output_format=output_format, output_file=output_file - ) + self.presenter.present(output_format=output_format, output_file=output_file) class CryptographyPresenter(AbstractPresenter): From e65f3968a9f93543fdfd23fe27ab6e4b9d5dd8ea Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 28 May 2025 10:02:00 +0200 Subject: [PATCH 329/489] [SP-2400] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8a24a64..e2e0b9ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.24.0] - 2025-05-06 +## [1.24.0] - 2025-05-28 ### Added - Add `crypto` subcommand to retrieve cryptographic algorithms for the given components - Add `crypto hints` subcommand to retrieve cryptographic hints for the given components From 6f7c09661e3e62bc9d446c00b769df3ba6ae33ce Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 2 Jun 2025 14:03:25 +0200 Subject: [PATCH 330/489] [SP-2655] Produce extra hash for windows line endings --- .github/workflows/python-local-test.yml | 85 +++++++-- CHANGELOG.md | 7 +- src/scanoss/__init__.py | 2 +- src/scanoss/winnowing.py | 85 ++++++++- tests/test_winnowing.py | 219 ++++++++++++++++++++++++ 5 files changed, 379 insertions(+), 19 deletions(-) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 4ac6b75a..ccc765bc 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -15,14 +15,18 @@ permissions: jobs: build: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ["3.10.x"] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.10.x" + python-version: ${{ matrix.python-version }} - name: Install Dependencies run: | @@ -39,18 +43,34 @@ jobs: retry_wait_seconds: 10 max_attempts: 3 retry_on: error + shell: bash command: | pip install -r requirements.txt - pip install dist/scanoss-*-py3-none-any.whl - which scanoss-py + # Install wheel with proper wildcard expansion + pip install dist/scanoss-*.whl + # Verify installation with platform-appropriate command + if command -v which &> /dev/null; then + which scanoss-py + else + where scanoss-py + fi - name: Run Tests + shell: bash run: | - which scanoss-py + if command -v which &> /dev/null; then + which scanoss-py + else + where scanoss-py + fi scanoss-py version scanoss-py utils fast scanoss-py scan tests > results.json - id_count=$(cat results.json | grep '"id":' | wc -l) + if [[ "$RUNNER_OS" == "Windows" ]]; then + id_count=$(findstr /C:"\"id\":" results.json | find /C ":" | tr -d ' ') + else + id_count=$(cat results.json | grep '"id":' | wc -l) + fi echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then echo "Error: Scan test did not produce any results. Failing" @@ -58,13 +78,22 @@ jobs: fi - name: Run Tests (fast winnowing) + shell: bash run: | pip install scanoss_winnowing - which scanoss-py + if command -v which &> /dev/null; then + which scanoss-py + else + where scanoss-py + fi scanoss-py version scanoss-py utils fast scanoss-py scan tests > results.json - id_count=$(cat results.json | grep '"id":' | wc -l) + if [[ "$RUNNER_OS" == "Windows" ]]; then + id_count=$(findstr /C:"\"id\":" results.json | find /C ":" | tr -d ' ') + else + id_count=$(cat results.json | grep '"id":' | wc -l) + fi echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then echo "Error: Scan test did not produce any results. Failing" @@ -72,13 +101,22 @@ jobs: fi - name: Run Tests HPSM (fast winnowing) + shell: bash run: | pip install scanoss_winnowing - which scanoss-py + if command -v which &> /dev/null; then + which scanoss-py + else + where scanoss-py + fi scanoss-py version scanoss-py utils fast scanoss-py wfp -H tests > fingers.wfp - wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + if [[ "$RUNNER_OS" == "Windows" ]]; then + wfp_count=$(findstr /C:"file=" fingers.wfp | find /C "file=" | tr -d ' ') + else + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + fi echo "WFP Count: $wfp_count" if [[ $wfp_count -lt 1 ]]; then echo "Error: WFP test did not produce any results. Failing" @@ -89,3 +127,30 @@ jobs: run: | python -m unittest + - name: Test Windows-specific functionality + if: runner.os == 'Windows' + shell: bash + run: | + echo "Testing Windows-specific fh2 hash generation..." + scanoss-py wfp tests > windows_fingerprints.wfp + if grep -q "fh2=" windows_fingerprints.wfp; then + echo "✓ Windows fh2 hash found in WFP output" + fh2_count=$(grep -c "fh2=" windows_fingerprints.wfp) + echo "Found $fh2_count fh2 hashes" + else + echo "✗ Error: No fh2 hashes found in Windows WFP output" + exit 1 + fi + + - name: Verify cross-platform consistency + if: runner.os != 'Windows' + shell: bash + run: | + echo "Testing Unix-specific behavior (no fh2)..." + scanoss-py wfp tests > unix_fingerprints.wfp + if grep -q "fh2=" unix_fingerprints.wfp; then + echo "✗ Error: Unexpected fh2 hash found in Unix WFP output" + exit 1 + else + echo "✓ No fh2 hashes found (expected for Unix systems)" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index e2e0b9ac..e21f3c1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +### [1.25.0] - 2025-06-02 +### Added +- Add `fh2` hash while fingerprinting mixed line ending files + ## [1.24.0] - 2025-05-28 ### Added - Add `crypto` subcommand to retrieve cryptographic algorithms for the given components @@ -522,4 +526,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.21.0]: https://github.com/scanoss/scanoss.py/compare/v1.20.6...v1.21.0 [1.22.0]: https://github.com/scanoss/scanoss.py/compare/v1.21.0...v1.22.0 [1.23.0]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.23.0 -[1.24.0]: https://github.com/scanoss/scanoss.py/compare/v1.23.0...v1.24.0 \ No newline at end of file +[1.24.0]: https://github.com/scanoss/scanoss.py/compare/v1.23.0...v1.24.0 +[1.25.0]: https://github.com/scanoss/scanoss.py/compare/v1.24.0...v1.25.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 228a61a1..629fead7 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.24.0' +__version__ = '1.25.0' diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 02953d73..07f522c3 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -33,8 +33,8 @@ import platform import re -from crc32c import crc32c from binaryornot.check import is_binary +from crc32c import crc32c from .scanossbase import ScanossBase @@ -157,7 +157,7 @@ class Winnowing(ScanossBase): a list of WFP fingerprints with their corresponding line numbers. """ - def __init__( + def __init__( # noqa: PLR0913 self, size_limit: bool = False, debug: bool = False, @@ -197,6 +197,7 @@ def __init__( self.strip_hpsm_ids = strip_hpsm_ids self.strip_snippet_ids = strip_snippet_ids self.hpsm = hpsm + self.is_windows = platform.system() == 'Windows' if hpsm: self.crc8_maxim_dow_table = [] self.crc8_generate_table() @@ -218,11 +219,11 @@ def __normalize(byte): return byte if byte >= ASCII_a: return byte - if (byte >= 65) and (byte <= 90): + if (byte >= ASCII_A) and (byte <= ASCII_Z): return byte + 32 return 0 - def __skip_snippets(self, file: str, src: str) -> bool: + def __skip_snippets(self, file: str, src: str) -> bool: # noqa: PLR0911 """ Determine files that are not of interest based on their content or file extension Parameters @@ -351,7 +352,71 @@ def __strip_snippets(self, file: str, wfp: str) -> str: self.print_debug(f'Stripped snippet ids from {file}') return wfp - def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: + def __detect_line_endings(self, contents: bytes) -> tuple[bool, bool, bool, bool]: + """Detect the types of line endings present in file contents. + + Args: + contents: File contents as bytes. + + Returns: + Tuple of (has_crlf, has_lf_only, has_cr_only, has_mixed) indicating which line ending types are present. + """ + has_crlf = b'\r\n' in contents + # For LF detection, we need to find LF that's not part of CRLF + content_without_crlf = contents.replace(b'\r\n', b'') + has_standalone_lf = b'\n' in content_without_crlf + # For CR detection, we need to find CR that's not part of CRLF + has_standalone_cr = b'\r' in content_without_crlf + + # Check if we have mixed line endings + line_ending_count = sum([has_crlf, has_standalone_lf, has_standalone_cr]) + has_mixed = line_ending_count > 1 + + return has_crlf, has_standalone_lf, has_standalone_cr, has_mixed + + def __calculate_opposite_line_ending_hash(self, contents: bytes) -> str: + """Calculate hash for contents with opposite line endings. + + If the file is primarily Unix (LF), calculates Windows (CRLF) hash. + If the file is primarily Windows (CRLF), calculates Unix (LF) hash. + + Args: + contents: File contents as bytes. + + Returns: + Hash with opposite line endings as hex string. + """ + has_crlf, has_standalone_lf, has_standalone_cr, has_mixed = self.__detect_line_endings(contents) + + # Normalize all line endings to LF first + normalized = contents.replace(b'\r\n', b'\n').replace(b'\r', b'\n') + + # Determine the dominant line ending type + if has_crlf and not has_standalone_lf and not has_standalone_cr: + # File is Windows (CRLF) - produce Unix (LF) hash + opposite_contents = normalized + else: + # File is Unix (LF/CR) or mixed - produce Windows (CRLF) hash + opposite_contents = normalized.replace(b'\n', b'\r\n') + + return hashlib.md5(opposite_contents).hexdigest() + + def __should_generate_opposite_hash(self, contents: bytes) -> bool: + """Determine if an opposite line ending hash (fh2) should be generated. + + Args: + contents: File contents as bytes. + + Returns: + True if fh2 hash should be generated, False otherwise. + """ + has_crlf, has_standalone_lf, has_standalone_cr, has_mixed = self.__detect_line_endings(contents) + + # Generate fh2 hash when file has any line endings (CRLF, LF, or CR) + # This allows us to always produce the opposite hash + return has_crlf or has_standalone_lf or has_standalone_cr + + def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # noqa: PLR0912, PLR0915 """ Generate a Winnowing fingerprint (WFP) for the given file contents Parameters @@ -371,7 +436,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: content_length = len(contents) original_filename = file - if platform.system() == 'Windows': + if self.is_windows: original_filename = file.replace('\\', '/') wfp_filename = repr(original_filename).strip("'") # return a utf-8 compatible version of the filename if self.obfuscate: # hide the real size of the file and its name, but keep the suffix @@ -380,6 +445,12 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: self.file_map[wfp_filename] = original_filename # Save the file name map for later (reverse lookup) wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename) + + # Add opposite line ending hash based on line ending analysis + if self.__should_generate_opposite_hash(contents): + opposite_hash = self.__calculate_opposite_line_ending_hash(contents) + wfp += f'fh2={opposite_hash}\n' + # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')): return wfp @@ -467,7 +538,7 @@ def calc_hpsm(self, content): for i, byte in enumerate(content): c = byte if c == ASCII_LF: # When there is a new line - if len(list_normalized): + if list_normalized: crc_lines.append(self.crc8_buffer(list_normalized)) list_normalized = [] elif last_line + 1 == i: diff --git a/tests/test_winnowing.py b/tests/test_winnowing.py index 59c6f0a3..1298de4b 100644 --- a/tests/test_winnowing.py +++ b/tests/test_winnowing.py @@ -23,6 +23,8 @@ """ import unittest +import platform +from unittest.mock import patch from scanoss.winnowing import Winnowing @@ -77,6 +79,223 @@ def test_snippet_strip(self): found = -1 self.assertEqual(found, -1) + def test_windows_hash_calculation(self): + """Test Windows-specific hash calculation with CRLF line endings.""" + import hashlib + + # Test content with LF line endings + content_lf = b'line1\nline2\nline3\n' + # Expected content with CRLF line endings for Windows hash + content_crlf = b'line1\r\nline2\r\nline3\r\n' + + # Calculate the expected Windows hash manually + expected_windows_hash = hashlib.md5(content_crlf).hexdigest() + lf_hash = hashlib.md5(content_lf).hexdigest() + + print(f'LF content hash: {lf_hash}') + print(f'CRLF content hash (Windows): {expected_windows_hash}') + + # They should be different + self.assertNotEqual(lf_hash, expected_windows_hash) + + @patch('platform.system') + def test_windows_wfp_includes_fh2(self, mock_platform): + """Test that WFP includes fh2 hash when running on Windows.""" + # Mock Windows environment + mock_platform.return_value = 'Windows' + winnowing = Winnowing(debug=True) + + filename = 'test-file.c' + content = b'int main() {\n return 0;\n}\n' + + wfp = winnowing.wfp_for_contents(filename, False, content) + + print(f'Windows WFP output:\n{wfp}') + + # Check that WFP contains fh2 line + self.assertIn('fh2=', wfp) + + # Extract the fh2 hash from WFP + lines = wfp.split('\n') + fh2_line = [line for line in lines if line.startswith('fh2=')] + self.assertEqual(len(fh2_line), 1) + + fh2_hash = fh2_line[0].split('=')[1] + + # Verify it matches expected CRLF conversion + import hashlib + content_crlf = content.replace(b'\n', b'\r\n') + expected_hash = hashlib.md5(content_crlf).hexdigest() + self.assertEqual(fh2_hash, expected_hash) + + @patch('platform.system') + def test_unix_wfp_excludes_fh2(self, mock_platform): + """Test that WFP does not include fh2 hash when running on Unix systems.""" + # Mock Unix environment + mock_platform.return_value = 'Linux' + winnowing = Winnowing(debug=True) + + filename = 'test-file.c' + content = b'int main() {\n return 0;\n}\n' + + wfp = winnowing.wfp_for_contents(filename, False, content) + + print(f'Unix WFP output:\n{wfp}') + + # Check that WFP does not contain fh2 line + self.assertNotIn('fh2=', wfp) + + def test_cross_platform_compatibility(self): + """Test that the same content produces consistent results across platforms.""" + filename = 'test-file.c' + content = b'#include \nint main() {\n printf("Hello World\\n");\n return 0;\n}\n' + + # Test with mocked Windows + with patch('platform.system', return_value='Windows'): + winnowing_windows = Winnowing(debug=True) + wfp_windows = winnowing_windows.wfp_for_contents(filename, False, content) + + # Test with mocked Linux + with patch('platform.system', return_value='Linux'): + winnowing_linux = Winnowing(debug=True) + wfp_linux = winnowing_linux.wfp_for_contents(filename, False, content) + + print(f'Windows WFP:\n{wfp_windows}') + print(f'Linux WFP:\n{wfp_linux}') + + # Both should have file line with same MD5 (original content) + windows_lines = wfp_windows.split('\n') + linux_lines = wfp_linux.split('\n') + + windows_file_line = [line for line in windows_lines if line.startswith('file=')][0] + linux_file_line = [line for line in linux_lines if line.startswith('file=')][0] + + # File lines should be identical (same original content MD5) + self.assertEqual(windows_file_line, linux_file_line) + + # Windows should have additional fh2 line + self.assertIn('fh2=', wfp_windows) + self.assertNotIn('fh2=', wfp_linux) + + # Extract snippets (everything after file/fh2 lines) + windows_snippets = [line for line in windows_lines if '=' in line and not line.startswith('file=') and not line.startswith('fh2=')] + linux_snippets = [line for line in linux_lines if '=' in line and not line.startswith('file=')] + + # Snippet fingerprints should be identical across platforms + self.assertEqual(windows_snippets, linux_snippets) + + def test_line_ending_detection(self): + """Test line ending detection logic.""" + winnowing = Winnowing(debug=True) + + # Test LF only + content_lf = b'line1\nline2\nline3\n' + has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_lf) + self.assertFalse(has_crlf) + self.assertTrue(has_lf) + self.assertFalse(has_cr) + self.assertFalse(has_mixed) + + # Test CRLF only + content_crlf = b'line1\r\nline2\r\nline3\r\n' + has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_crlf) + self.assertTrue(has_crlf) + self.assertFalse(has_lf) + self.assertFalse(has_cr) + self.assertFalse(has_mixed) + + # Test CR only (old Mac style) + content_cr = b'line1\rline2\rline3\r' + has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_cr) + self.assertFalse(has_crlf) + self.assertFalse(has_lf) + self.assertTrue(has_cr) + self.assertFalse(has_mixed) + + # Test mixed CRLF and LF + content_mixed = b'line1\r\nline2\nline3\r\n' + has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_mixed) + self.assertTrue(has_crlf) + self.assertTrue(has_lf) + self.assertFalse(has_cr) + self.assertTrue(has_mixed) + + def test_mixed_line_endings_scenarios(self): + """Test various mixed line ending scenarios.""" + filename = 'test-file.c' + + # Test 1: LF only on Windows (should generate fh2) + with patch('platform.system', return_value='Windows'): + winnowing = Winnowing(debug=True) + content_lf = b'int main() {\n return 0;\n}\n' + wfp = winnowing.wfp_for_contents(filename, False, content_lf) + self.assertIn('fh2=', wfp) + print(f'LF on Windows - WFP includes fh2: ✓') + + # Test 2: CRLF only on Windows (should NOT generate fh2) + with patch('platform.system', return_value='Windows'): + winnowing = Winnowing(debug=True) + content_crlf = b'int main() {\r\n return 0;\r\n}\r\n' + wfp = winnowing.wfp_for_contents(filename, False, content_crlf) + self.assertNotIn('fh2=', wfp) + print(f'CRLF on Windows - WFP excludes fh2: ✓') + + # Test 3: Mixed line endings on any OS (should generate fh2) + with patch('platform.system', return_value='Linux'): + winnowing = Winnowing(debug=True) + content_mixed = b'int main() {\r\n printf("hello");\n return 0;\r\n}\n' + wfp = winnowing.wfp_for_contents(filename, False, content_mixed) + self.assertIn('fh2=', wfp) + print(f'Mixed line endings on Linux - WFP includes fh2: ✓') + + # Test 4: CR only on Windows (should generate fh2) + with patch('platform.system', return_value='Windows'): + winnowing = Winnowing(debug=True) + content_cr = b'int main() {\r return 0;\r}\r' + wfp = winnowing.wfp_for_contents(filename, False, content_cr) + self.assertIn('fh2=', wfp) + print(f'CR only on Windows - WFP includes fh2: ✓') + + def test_windows_hash_normalization(self): + """Test that Windows hash properly normalizes different line endings.""" + winnowing = Winnowing(debug=True) + + # All these should produce the same Windows hash after normalization + content_lf = b'line1\nline2\nline3\n' + content_crlf = b'line1\r\nline2\r\nline3\r\n' + content_cr = b'line1\rline2\rline3\r' + content_mixed = b'line1\r\nline2\nline3\r' + + hash_lf = winnowing.__calculate_opposite_line_ending_hash(content_lf) + hash_crlf = winnowing.__calculate_opposite_line_ending_hash(content_crlf) + hash_cr = winnowing.__calculate_opposite_line_ending_hash(content_cr) + hash_mixed = winnowing.__calculate_opposite_line_ending_hash(content_mixed) + + print(f'LF hash: {hash_lf}') + print(f'CRLF hash: {hash_crlf}') + print(f'CR hash: {hash_cr}') + print(f'Mixed hash: {hash_mixed}') + + # All should be equal after normalization + self.assertEqual(hash_lf, hash_crlf) + self.assertEqual(hash_lf, hash_cr) + self.assertEqual(hash_lf, hash_mixed) + + @unittest.skipUnless(platform.system() == 'Windows', 'Windows-specific test') + def test_actual_windows_behavior(self): + """Test actual Windows behavior when running on Windows.""" + winnowing = Winnowing(debug=True) + filename = 'test-file.c' + content = b'int main() {\n return 0;\n}\n' + + wfp = winnowing.wfp_for_contents(filename, False, content) + + print(f'Actual Windows WFP:\n{wfp}') + + # On actual Windows with LF content, should include fh2 + if platform.system() == 'Windows': + self.assertIn('fh2=', wfp) + if __name__ == '__main__': unittest.main() From 9118049cc29cabd4a1ad6de0760af24fd097ceec Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Jun 2025 09:06:14 +0200 Subject: [PATCH 331/489] [SP-2655] Fix tests --- tests/test_winnowing.py | 92 ----------------------------------------- 1 file changed, 92 deletions(-) diff --git a/tests/test_winnowing.py b/tests/test_winnowing.py index 1298de4b..3ea8e579 100644 --- a/tests/test_winnowing.py +++ b/tests/test_winnowing.py @@ -128,62 +128,6 @@ def test_windows_wfp_includes_fh2(self, mock_platform): expected_hash = hashlib.md5(content_crlf).hexdigest() self.assertEqual(fh2_hash, expected_hash) - @patch('platform.system') - def test_unix_wfp_excludes_fh2(self, mock_platform): - """Test that WFP does not include fh2 hash when running on Unix systems.""" - # Mock Unix environment - mock_platform.return_value = 'Linux' - winnowing = Winnowing(debug=True) - - filename = 'test-file.c' - content = b'int main() {\n return 0;\n}\n' - - wfp = winnowing.wfp_for_contents(filename, False, content) - - print(f'Unix WFP output:\n{wfp}') - - # Check that WFP does not contain fh2 line - self.assertNotIn('fh2=', wfp) - - def test_cross_platform_compatibility(self): - """Test that the same content produces consistent results across platforms.""" - filename = 'test-file.c' - content = b'#include \nint main() {\n printf("Hello World\\n");\n return 0;\n}\n' - - # Test with mocked Windows - with patch('platform.system', return_value='Windows'): - winnowing_windows = Winnowing(debug=True) - wfp_windows = winnowing_windows.wfp_for_contents(filename, False, content) - - # Test with mocked Linux - with patch('platform.system', return_value='Linux'): - winnowing_linux = Winnowing(debug=True) - wfp_linux = winnowing_linux.wfp_for_contents(filename, False, content) - - print(f'Windows WFP:\n{wfp_windows}') - print(f'Linux WFP:\n{wfp_linux}') - - # Both should have file line with same MD5 (original content) - windows_lines = wfp_windows.split('\n') - linux_lines = wfp_linux.split('\n') - - windows_file_line = [line for line in windows_lines if line.startswith('file=')][0] - linux_file_line = [line for line in linux_lines if line.startswith('file=')][0] - - # File lines should be identical (same original content MD5) - self.assertEqual(windows_file_line, linux_file_line) - - # Windows should have additional fh2 line - self.assertIn('fh2=', wfp_windows) - self.assertNotIn('fh2=', wfp_linux) - - # Extract snippets (everything after file/fh2 lines) - windows_snippets = [line for line in windows_lines if '=' in line and not line.startswith('file=') and not line.startswith('fh2=')] - linux_snippets = [line for line in linux_lines if '=' in line and not line.startswith('file=')] - - # Snippet fingerprints should be identical across platforms - self.assertEqual(windows_snippets, linux_snippets) - def test_line_ending_detection(self): """Test line ending detection logic.""" winnowing = Winnowing(debug=True) @@ -220,42 +164,6 @@ def test_line_ending_detection(self): self.assertFalse(has_cr) self.assertTrue(has_mixed) - def test_mixed_line_endings_scenarios(self): - """Test various mixed line ending scenarios.""" - filename = 'test-file.c' - - # Test 1: LF only on Windows (should generate fh2) - with patch('platform.system', return_value='Windows'): - winnowing = Winnowing(debug=True) - content_lf = b'int main() {\n return 0;\n}\n' - wfp = winnowing.wfp_for_contents(filename, False, content_lf) - self.assertIn('fh2=', wfp) - print(f'LF on Windows - WFP includes fh2: ✓') - - # Test 2: CRLF only on Windows (should NOT generate fh2) - with patch('platform.system', return_value='Windows'): - winnowing = Winnowing(debug=True) - content_crlf = b'int main() {\r\n return 0;\r\n}\r\n' - wfp = winnowing.wfp_for_contents(filename, False, content_crlf) - self.assertNotIn('fh2=', wfp) - print(f'CRLF on Windows - WFP excludes fh2: ✓') - - # Test 3: Mixed line endings on any OS (should generate fh2) - with patch('platform.system', return_value='Linux'): - winnowing = Winnowing(debug=True) - content_mixed = b'int main() {\r\n printf("hello");\n return 0;\r\n}\n' - wfp = winnowing.wfp_for_contents(filename, False, content_mixed) - self.assertIn('fh2=', wfp) - print(f'Mixed line endings on Linux - WFP includes fh2: ✓') - - # Test 4: CR only on Windows (should generate fh2) - with patch('platform.system', return_value='Windows'): - winnowing = Winnowing(debug=True) - content_cr = b'int main() {\r return 0;\r}\r' - wfp = winnowing.wfp_for_contents(filename, False, content_cr) - self.assertIn('fh2=', wfp) - print(f'CR only on Windows - WFP includes fh2: ✓') - def test_windows_hash_normalization(self): """Test that Windows hash properly normalizes different line endings.""" winnowing = Winnowing(debug=True) From e7e8a3e4fb0fefda0d09b40a30127ef4bd07b3f2 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Jun 2025 09:37:41 +0200 Subject: [PATCH 332/489] [SP-2655] Fix tests --- tests/test_winnowing.py | 226 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 207 insertions(+), 19 deletions(-) diff --git a/tests/test_winnowing.py b/tests/test_winnowing.py index 3ea8e579..bf1baa83 100644 --- a/tests/test_winnowing.py +++ b/tests/test_winnowing.py @@ -134,7 +134,7 @@ def test_line_ending_detection(self): # Test LF only content_lf = b'line1\nline2\nline3\n' - has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_lf) + has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_lf) self.assertFalse(has_crlf) self.assertTrue(has_lf) self.assertFalse(has_cr) @@ -142,7 +142,7 @@ def test_line_ending_detection(self): # Test CRLF only content_crlf = b'line1\r\nline2\r\nline3\r\n' - has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_crlf) + has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_crlf) self.assertTrue(has_crlf) self.assertFalse(has_lf) self.assertFalse(has_cr) @@ -150,7 +150,7 @@ def test_line_ending_detection(self): # Test CR only (old Mac style) content_cr = b'line1\rline2\rline3\r' - has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_cr) + has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_cr) self.assertFalse(has_crlf) self.assertFalse(has_lf) self.assertTrue(has_cr) @@ -158,37 +158,39 @@ def test_line_ending_detection(self): # Test mixed CRLF and LF content_mixed = b'line1\r\nline2\nline3\r\n' - has_crlf, has_lf, has_cr, has_mixed = winnowing.__detect_line_endings(content_mixed) + has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_mixed) self.assertTrue(has_crlf) self.assertTrue(has_lf) self.assertFalse(has_cr) self.assertTrue(has_mixed) - def test_windows_hash_normalization(self): - """Test that Windows hash properly normalizes different line endings.""" + def test_opposite_hash_logic(self): + """Test the logic of opposite hash calculation.""" winnowing = Winnowing(debug=True) - # All these should produce the same Windows hash after normalization + # Test different line ending scenarios content_lf = b'line1\nline2\nline3\n' content_crlf = b'line1\r\nline2\r\nline3\r\n' content_cr = b'line1\rline2\rline3\r' content_mixed = b'line1\r\nline2\nline3\r' - hash_lf = winnowing.__calculate_opposite_line_ending_hash(content_lf) - hash_crlf = winnowing.__calculate_opposite_line_ending_hash(content_crlf) - hash_cr = winnowing.__calculate_opposite_line_ending_hash(content_cr) - hash_mixed = winnowing.__calculate_opposite_line_ending_hash(content_mixed) + hash_lf = winnowing._Winnowing__calculate_opposite_line_ending_hash(content_lf) + hash_crlf = winnowing._Winnowing__calculate_opposite_line_ending_hash(content_crlf) + hash_cr = winnowing._Winnowing__calculate_opposite_line_ending_hash(content_cr) + hash_mixed = winnowing._Winnowing__calculate_opposite_line_ending_hash(content_mixed) - print(f'LF hash: {hash_lf}') - print(f'CRLF hash: {hash_crlf}') - print(f'CR hash: {hash_cr}') - print(f'Mixed hash: {hash_mixed}') + print(f'LF opposite hash: {hash_lf}') + print(f'CRLF opposite hash: {hash_crlf}') + print(f'CR opposite hash: {hash_cr}') + print(f'Mixed opposite hash: {hash_mixed}') - # All should be equal after normalization - self.assertEqual(hash_lf, hash_crlf) + # LF, CR, and mixed content should all produce CRLF hash (same result) self.assertEqual(hash_lf, hash_cr) self.assertEqual(hash_lf, hash_mixed) + # CRLF content should produce LF hash (different from the others) + self.assertNotEqual(hash_crlf, hash_lf) + @unittest.skipUnless(platform.system() == 'Windows', 'Windows-specific test') def test_actual_windows_behavior(self): """Test actual Windows behavior when running on Windows.""" @@ -201,8 +203,194 @@ def test_actual_windows_behavior(self): print(f'Actual Windows WFP:\n{wfp}') # On actual Windows with LF content, should include fh2 - if platform.system() == 'Windows': - self.assertIn('fh2=', wfp) + # Should always generate fh2 when line endings are present + self.assertIn('fh2=', wfp) + + def test_empty_file_fh2(self): + """Test fh2 behavior with empty files.""" + winnowing = Winnowing(debug=True) + content = b'' + wfp = winnowing.wfp_for_contents('empty.txt', False, content) + + print(f'Empty file WFP:\n{wfp}') + + # Empty files should not generate fh2 + self.assertNotIn('fh2=', wfp) + + def test_no_line_endings_fh2(self): + """Test files without any line endings.""" + winnowing = Winnowing(debug=True) + content = b'no line endings here' + wfp = winnowing.wfp_for_contents('noline.txt', False, content) + + print(f'No line endings WFP:\n{wfp}') + + # Files without line endings should not generate fh2 + self.assertNotIn('fh2=', wfp) + + def test_all_platforms_generate_fh2(self): + """Test that all platforms generate fh2 when line endings are present.""" + winnowing = Winnowing(debug=True) + content = b'line1\nline2\n' + wfp = winnowing.wfp_for_contents('test.txt', False, content) + + print(f'Platform-independent WFP:\n{wfp}') + + # Any platform should generate fh2 when line endings are present + self.assertIn('fh2=', wfp) + + def test_verify_opposite_hash_calculation(self): + """Test that the opposite hash calculation works correctly.""" + winnowing = Winnowing(debug=True) + + # Test LF -> CRLF conversion + content_lf = b'line1\nline2\nline3\n' + wfp_lf = winnowing.wfp_for_contents('test_lf.txt', False, content_lf) + + # Test CRLF -> LF conversion + content_crlf = b'line1\r\nline2\r\nline3\r\n' + wfp_crlf = winnowing.wfp_for_contents('test_crlf.txt', False, content_crlf) + + print(f'LF content WFP:\n{wfp_lf}') + print(f'CRLF content WFP:\n{wfp_crlf}') + + # Both should generate fh2 + self.assertIn('fh2=', wfp_lf) + self.assertIn('fh2=', wfp_crlf) + + # Extract fh2 values + lf_fh2 = wfp_lf.split('fh2=')[1].split('\n')[0] + crlf_fh2 = wfp_crlf.split('fh2=')[1].split('\n')[0] + + # The fh2 values should be swapped (LF file gets CRLF hash, CRLF file gets LF hash) + import hashlib + expected_lf_to_crlf = hashlib.md5(content_lf.replace(b'\n', b'\r\n')).hexdigest() + expected_crlf_to_lf = hashlib.md5(content_crlf.replace(b'\r\n', b'\n')).hexdigest() + + self.assertEqual(lf_fh2, expected_lf_to_crlf) + self.assertEqual(crlf_fh2, expected_crlf_to_lf) + + def test_binary_file_with_line_endings(self): + """Test binary files with embedded line endings.""" + winnowing = Winnowing(debug=True) + # Binary content with embedded newlines + content = b'\x00\x01\n\x02\x03\r\n\x04' + wfp = winnowing.wfp_for_contents('binary.bin', True, content) + + print(f'Binary file WFP:\n{wfp}') + + # Binary files should still generate fh2 if they have line endings (platform independent) + self.assertIn('fh2=', wfp) + + def test_cr_only_line_endings(self): + """Test classic Mac CR-only line endings.""" + winnowing = Winnowing(debug=True) + content = b'line1\rline2\rline3\r' + wfp = winnowing.wfp_for_contents('mac.txt', False, content) + + print(f'CR-only WFP:\n{wfp}') + + # Should generate fh2 (platform independent) + self.assertIn('fh2=', wfp) + + # Should normalize CR to CRLF for the opposite hash + import hashlib + expected = content.replace(b'\r', b'\r\n') + expected_hash = hashlib.md5(expected).hexdigest() + self.assertIn(f'fh2={expected_hash}', wfp) + + def test_whitespace_only_file(self): + """Test files with only whitespace characters.""" + winnowing = Winnowing(debug=True) + content = b' \n\t\n \n' + wfp = winnowing.wfp_for_contents('whitespace.txt', False, content) + + print(f'Whitespace-only WFP:\n{wfp}') + + # Should generate fh2 since it has line endings + self.assertIn('fh2=', wfp) + + def test_mixed_complex_line_endings(self): + """Test complex mixed line ending scenarios.""" + winnowing = Winnowing(debug=True) + # Mix of CRLF, LF, and CR + content = b'line1\r\nline2\nline3\rline4\r\nline5\n' + wfp = winnowing.wfp_for_contents('mixed.txt', False, content) + + print(f'Mixed line endings WFP:\n{wfp}') + + # Should generate fh2 + self.assertIn('fh2=', wfp) + + # Verify the hash calculation + import hashlib + normalized = content.replace(b'\r\n', b'\n').replace(b'\r', b'\n') + expected_crlf = normalized.replace(b'\n', b'\r\n') + expected_hash = hashlib.md5(expected_crlf).hexdigest() + self.assertIn(f'fh2={expected_hash}', wfp) + + def test_fh2_with_skip_snippets(self): + """Test fh2 generation when skip_snippets is enabled.""" + winnowing = Winnowing(debug=True, skip_snippets=True) + content = b'line1\nline2\nline3\n' + wfp = winnowing.wfp_for_contents('test.txt', False, content) + + print(f'Skip snippets WFP:\n{wfp}') + + # Should still generate fh2 even when skipping snippets + self.assertIn('fh2=', wfp) + # But should not contain snippet fingerprints (line numbers) + lines = wfp.split('\n') + snippet_lines = [line for line in lines if '=' in line and line[0].isdigit()] + self.assertEqual(len(snippet_lines), 0) + + def test_fh2_with_obfuscation(self): + """Test fh2 generation with obfuscation enabled.""" + winnowing = Winnowing(debug=True, obfuscate=True) + content = b'line1\nline2\nline3\n' + wfp = winnowing.wfp_for_contents('test.txt', False, content) + + print(f'Obfuscated WFP:\n{wfp}') + + # Should still generate fh2 with obfuscation + self.assertIn('fh2=', wfp) + # Filename should be obfuscated + self.assertIn('1.txt', wfp) + self.assertNotIn('test.txt', wfp) + + def test_large_file_with_line_endings(self): + """Test large files with many line endings.""" + winnowing = Winnowing(debug=True, size_limit=True, post_size=1) # 1KB limit + # Create content larger than the limit + content = b'line\n' * 1000 # Should exceed 1KB + wfp = winnowing.wfp_for_contents('large.txt', False, content) + + print(f'Large file WFP length: {len(wfp)}') + + # Should still generate fh2 even with size limits + self.assertIn('fh2=', wfp) + + def test_single_line_no_newline(self): + """Test single line files without trailing newline.""" + winnowing = Winnowing(debug=True) + content = b'single line without newline' + wfp = winnowing.wfp_for_contents('single.txt', False, content) + + print(f'Single line no newline WFP:\n{wfp}') + + # Should not generate fh2 (no line endings) + self.assertNotIn('fh2=', wfp) + + def test_file_with_null_bytes_and_newlines(self): + """Test files with null bytes mixed with newlines.""" + winnowing = Winnowing(debug=True) + content = b'line1\x00\nline2\x00\x00\nline3\n' + wfp = winnowing.wfp_for_contents('nullbytes.txt', False, content) + + print(f'Null bytes with newlines WFP:\n{wfp}') + + # Should generate fh2 (has line endings) + self.assertIn('fh2=', wfp) if __name__ == '__main__': From 3156d42a36df3549b5a28c0cb08e5fbdca7d8853 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Jun 2025 09:52:38 +0200 Subject: [PATCH 333/489] [SP-2665] Update workflow --- .github/workflows/python-local-test.yml | 84 +++++-------------------- 1 file changed, 16 insertions(+), 68 deletions(-) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index ccc765bc..e10e6d83 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -15,18 +15,14 @@ permissions: jobs: build: - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.10.x"] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.10.x" - name: Install Dependencies run: | @@ -46,31 +42,16 @@ jobs: shell: bash command: | pip install -r requirements.txt - # Install wheel with proper wildcard expansion pip install dist/scanoss-*.whl - # Verify installation with platform-appropriate command - if command -v which &> /dev/null; then - which scanoss-py - else - where scanoss-py - fi + which scanoss-py - name: Run Tests - shell: bash run: | - if command -v which &> /dev/null; then - which scanoss-py - else - where scanoss-py - fi + which scanoss-py scanoss-py version scanoss-py utils fast scanoss-py scan tests > results.json - if [[ "$RUNNER_OS" == "Windows" ]]; then - id_count=$(findstr /C:"\"id\":" results.json | find /C ":" | tr -d ' ') - else - id_count=$(cat results.json | grep '"id":' | wc -l) - fi + id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then echo "Error: Scan test did not produce any results. Failing" @@ -78,22 +59,13 @@ jobs: fi - name: Run Tests (fast winnowing) - shell: bash run: | pip install scanoss_winnowing - if command -v which &> /dev/null; then - which scanoss-py - else - where scanoss-py - fi + which scanoss-py scanoss-py version scanoss-py utils fast scanoss-py scan tests > results.json - if [[ "$RUNNER_OS" == "Windows" ]]; then - id_count=$(findstr /C:"\"id\":" results.json | find /C ":" | tr -d ' ') - else - id_count=$(cat results.json | grep '"id":' | wc -l) - fi + id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then echo "Error: Scan test did not produce any results. Failing" @@ -101,22 +73,13 @@ jobs: fi - name: Run Tests HPSM (fast winnowing) - shell: bash run: | pip install scanoss_winnowing - if command -v which &> /dev/null; then - which scanoss-py - else - where scanoss-py - fi + which scanoss-py scanoss-py version scanoss-py utils fast scanoss-py wfp -H tests > fingers.wfp - if [[ "$RUNNER_OS" == "Windows" ]]; then - wfp_count=$(findstr /C:"file=" fingers.wfp | find /C "file=" | tr -d ' ') - else - wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) - fi + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) echo "WFP Count: $wfp_count" if [[ $wfp_count -lt 1 ]]; then echo "Error: WFP test did not produce any results. Failing" @@ -127,30 +90,15 @@ jobs: run: | python -m unittest - - name: Test Windows-specific functionality - if: runner.os == 'Windows' - shell: bash + - name: Test fh2 hash generation run: | - echo "Testing Windows-specific fh2 hash generation..." - scanoss-py wfp tests > windows_fingerprints.wfp - if grep -q "fh2=" windows_fingerprints.wfp; then - echo "✓ Windows fh2 hash found in WFP output" - fh2_count=$(grep -c "fh2=" windows_fingerprints.wfp) + echo "Testing fh2 hash generation for files with line endings..." + scanoss-py wfp tests > fingerprints.wfp + if grep -q "fh2=" fingerprints.wfp; then + echo "✓ fh2 hashes found in WFP output" + fh2_count=$(grep -c "fh2=" fingerprints.wfp) echo "Found $fh2_count fh2 hashes" else - echo "✗ Error: No fh2 hashes found in Windows WFP output" + echo "✗ Error: No fh2 hashes found in WFP output" exit 1 fi - - - name: Verify cross-platform consistency - if: runner.os != 'Windows' - shell: bash - run: | - echo "Testing Unix-specific behavior (no fh2)..." - scanoss-py wfp tests > unix_fingerprints.wfp - if grep -q "fh2=" unix_fingerprints.wfp; then - echo "✗ Error: Unexpected fh2 hash found in Unix WFP output" - exit 1 - else - echo "✓ No fh2 hashes found (expected for Unix systems)" - fi From 463242c458de0dc78c3c469b9f74fcd73da5aee0 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 3 Jun 2025 13:31:58 +0200 Subject: [PATCH 334/489] [SP-2665] Update workflow --- .github/workflows/python-local-test.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index e10e6d83..12b001b3 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -90,15 +90,3 @@ jobs: run: | python -m unittest - - name: Test fh2 hash generation - run: | - echo "Testing fh2 hash generation for files with line endings..." - scanoss-py wfp tests > fingerprints.wfp - if grep -q "fh2=" fingerprints.wfp; then - echo "✓ fh2 hashes found in WFP output" - fh2_count=$(grep -c "fh2=" fingerprints.wfp) - echo "Found $fh2_count fh2 hashes" - else - echo "✗ Error: No fh2 hashes found in WFP output" - exit 1 - fi From a32f4041719631e04be88a08ef8ee7cc23c6987f Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 5 Jun 2025 10:16:59 +0200 Subject: [PATCH 335/489] [SP-2655] fix type error --- src/scanoss/winnowing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 07f522c3..905096e0 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -32,6 +32,7 @@ import pathlib import platform import re +from typing import Tuple from binaryornot.check import is_binary from crc32c import crc32c @@ -352,7 +353,7 @@ def __strip_snippets(self, file: str, wfp: str) -> str: self.print_debug(f'Stripped snippet ids from {file}') return wfp - def __detect_line_endings(self, contents: bytes) -> tuple[bool, bool, bool, bool]: + def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool, bool]: """Detect the types of line endings present in file contents. Args: From 93ef66f8085a8c72855dc352935595f4b4c90aa9 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 9 Jun 2025 16:39:16 +0200 Subject: [PATCH 336/489] [SP-2655] Only produce fh2 hash if we detect line endings and is not bin file --- src/scanoss/winnowing.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 905096e0..bae904cb 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -355,10 +355,10 @@ def __strip_snippets(self, file: str, wfp: str) -> str: def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool, bool]: """Detect the types of line endings present in file contents. - + Args: contents: File contents as bytes. - + Returns: Tuple of (has_crlf, has_lf_only, has_cr_only, has_mixed) indicating which line ending types are present. """ @@ -368,16 +368,16 @@ def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool, bool has_standalone_lf = b'\n' in content_without_crlf # For CR detection, we need to find CR that's not part of CRLF has_standalone_cr = b'\r' in content_without_crlf - + # Check if we have mixed line endings line_ending_count = sum([has_crlf, has_standalone_lf, has_standalone_cr]) has_mixed = line_ending_count > 1 - + return has_crlf, has_standalone_lf, has_standalone_cr, has_mixed - def __calculate_opposite_line_ending_hash(self, contents: bytes) -> str: + def __calculate_opposite_line_ending_hash(self, contents: bytes): """Calculate hash for contents with opposite line endings. - + If the file is primarily Unix (LF), calculates Windows (CRLF) hash. If the file is primarily Windows (CRLF), calculates Unix (LF) hash. @@ -385,34 +385,37 @@ def __calculate_opposite_line_ending_hash(self, contents: bytes) -> str: contents: File contents as bytes. Returns: - Hash with opposite line endings as hex string. + Hash with opposite line endings as hex string, or None if no line endings detected. """ has_crlf, has_standalone_lf, has_standalone_cr, has_mixed = self.__detect_line_endings(contents) - + + if not has_crlf and not has_standalone_lf and not has_standalone_cr: + return None + # Normalize all line endings to LF first normalized = contents.replace(b'\r\n', b'\n').replace(b'\r', b'\n') - + # Determine the dominant line ending type if has_crlf and not has_standalone_lf and not has_standalone_cr: # File is Windows (CRLF) - produce Unix (LF) hash opposite_contents = normalized else: - # File is Unix (LF/CR) or mixed - produce Windows (CRLF) hash + # File is Unix (LF/CR) or mixed - produce Windows (CRLF) hash opposite_contents = normalized.replace(b'\n', b'\r\n') - + return hashlib.md5(opposite_contents).hexdigest() def __should_generate_opposite_hash(self, contents: bytes) -> bool: """Determine if an opposite line ending hash (fh2) should be generated. - + Args: contents: File contents as bytes. - + Returns: True if fh2 hash should be generated, False otherwise. """ - has_crlf, has_standalone_lf, has_standalone_cr, has_mixed = self.__detect_line_endings(contents) - + has_crlf, has_standalone_lf, has_standalone_cr = self.__detect_line_endings(contents) + # Generate fh2 hash when file has any line endings (CRLF, LF, or CR) # This allows us to always produce the opposite hash return has_crlf or has_standalone_lf or has_standalone_cr @@ -448,9 +451,10 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename) # Add opposite line ending hash based on line ending analysis - if self.__should_generate_opposite_hash(contents): + if not bin_file and self.__should_generate_opposite_hash(contents): opposite_hash = self.__calculate_opposite_line_ending_hash(contents) - wfp += f'fh2={opposite_hash}\n' + if opposite_hash is not None: + wfp += f'fh2={opposite_hash}\n' # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')): From 89668484edf09efeb337976819981d255ddb61b9 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 9 Jun 2025 16:42:22 +0200 Subject: [PATCH 337/489] [SP-2655] Fix tests --- tests/test_winnowing.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/test_winnowing.py b/tests/test_winnowing.py index bf1baa83..a261830f 100644 --- a/tests/test_winnowing.py +++ b/tests/test_winnowing.py @@ -22,8 +22,8 @@ THE SOFTWARE. """ -import unittest import platform +import unittest from unittest.mock import patch from scanoss.winnowing import Winnowing @@ -134,35 +134,31 @@ def test_line_ending_detection(self): # Test LF only content_lf = b'line1\nline2\nline3\n' - has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_lf) + has_crlf, has_lf, has_cr = winnowing._Winnowing__detect_line_endings(content_lf) self.assertFalse(has_crlf) self.assertTrue(has_lf) self.assertFalse(has_cr) - self.assertFalse(has_mixed) # Test CRLF only content_crlf = b'line1\r\nline2\r\nline3\r\n' - has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_crlf) + has_crlf, has_lf, has_cr = winnowing._Winnowing__detect_line_endings(content_crlf) self.assertTrue(has_crlf) self.assertFalse(has_lf) self.assertFalse(has_cr) - self.assertFalse(has_mixed) # Test CR only (old Mac style) content_cr = b'line1\rline2\rline3\r' - has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_cr) + has_crlf, has_lf, has_cr = winnowing._Winnowing__detect_line_endings(content_cr) self.assertFalse(has_crlf) self.assertFalse(has_lf) self.assertTrue(has_cr) - self.assertFalse(has_mixed) # Test mixed CRLF and LF content_mixed = b'line1\r\nline2\nline3\r\n' - has_crlf, has_lf, has_cr, has_mixed = winnowing._Winnowing__detect_line_endings(content_mixed) + has_crlf, has_lf, has_cr = winnowing._Winnowing__detect_line_endings(content_mixed) self.assertTrue(has_crlf) self.assertTrue(has_lf) self.assertFalse(has_cr) - self.assertTrue(has_mixed) def test_opposite_hash_logic(self): """Test the logic of opposite hash calculation.""" From ede0477f3ea1b13a0147154b565b1bf6a72a6843 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 9 Jun 2025 19:43:18 +0200 Subject: [PATCH 338/489] [SP-2655] Fix tests and optimize performance --- src/scanoss/winnowing.py | 27 ++++----------------------- tests/test_winnowing.py | 4 ++-- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index bae904cb..9b2e3a57 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -353,7 +353,7 @@ def __strip_snippets(self, file: str, wfp: str) -> str: self.print_debug(f'Stripped snippet ids from {file}') return wfp - def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool, bool]: + def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool]: """Detect the types of line endings present in file contents. Args: @@ -369,11 +369,7 @@ def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool, bool # For CR detection, we need to find CR that's not part of CRLF has_standalone_cr = b'\r' in content_without_crlf - # Check if we have mixed line endings - line_ending_count = sum([has_crlf, has_standalone_lf, has_standalone_cr]) - has_mixed = line_ending_count > 1 - - return has_crlf, has_standalone_lf, has_standalone_cr, has_mixed + return has_crlf, has_standalone_lf, has_standalone_cr def __calculate_opposite_line_ending_hash(self, contents: bytes): """Calculate hash for contents with opposite line endings. @@ -387,7 +383,7 @@ def __calculate_opposite_line_ending_hash(self, contents: bytes): Returns: Hash with opposite line endings as hex string, or None if no line endings detected. """ - has_crlf, has_standalone_lf, has_standalone_cr, has_mixed = self.__detect_line_endings(contents) + has_crlf, has_standalone_lf, has_standalone_cr = self.__detect_line_endings(contents) if not has_crlf and not has_standalone_lf and not has_standalone_cr: return None @@ -405,21 +401,6 @@ def __calculate_opposite_line_ending_hash(self, contents: bytes): return hashlib.md5(opposite_contents).hexdigest() - def __should_generate_opposite_hash(self, contents: bytes) -> bool: - """Determine if an opposite line ending hash (fh2) should be generated. - - Args: - contents: File contents as bytes. - - Returns: - True if fh2 hash should be generated, False otherwise. - """ - has_crlf, has_standalone_lf, has_standalone_cr = self.__detect_line_endings(contents) - - # Generate fh2 hash when file has any line endings (CRLF, LF, or CR) - # This allows us to always produce the opposite hash - return has_crlf or has_standalone_lf or has_standalone_cr - def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # noqa: PLR0912, PLR0915 """ Generate a Winnowing fingerprint (WFP) for the given file contents @@ -451,7 +432,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename) # Add opposite line ending hash based on line ending analysis - if not bin_file and self.__should_generate_opposite_hash(contents): + if not bin_file: opposite_hash = self.__calculate_opposite_line_ending_hash(contents) if opposite_hash is not None: wfp += f'fh2={opposite_hash}\n' diff --git a/tests/test_winnowing.py b/tests/test_winnowing.py index a261830f..cc26947d 100644 --- a/tests/test_winnowing.py +++ b/tests/test_winnowing.py @@ -275,8 +275,8 @@ def test_binary_file_with_line_endings(self): print(f'Binary file WFP:\n{wfp}') - # Binary files should still generate fh2 if they have line endings (platform independent) - self.assertIn('fh2=', wfp) + # Binary files should not generate fh2 + self.assertNotIn('fh2=', wfp) def test_cr_only_line_endings(self): """Test classic Mac CR-only line endings.""" From ac009ff1e819bbf392c173f7757a111c54af7e8e Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Tue, 10 Jun 2025 04:56:46 -0300 Subject: [PATCH 339/489] chore:SP-2675 Displays warning messages when debug flag is enabled (#122) * chore:SP-2675 Displays warning messages when debug flag is enabled --- src/scanoss/inspection/policy_check.py | 124 +++++++++++++++---------- 1 file changed, 77 insertions(+), 47 deletions(-) diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index a7e352c8..e60b2974 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -26,9 +26,10 @@ import os.path from abc import abstractmethod from enum import Enum -from typing import Callable, List, Dict, Any -from .utils.license_utils import LicenseUtil +from typing import Any, Callable, Dict, List + from ..scanossbase import ScanossBase +from .utils.license_utils import LicenseUtil class PolicyStatus(Enum): @@ -87,7 +88,7 @@ class PolicyCheck(ScanossBase): VALID_FORMATS = {'md', 'json', 'jira_md'} - def __init__( + def __init__( # noqa: PLR0913 self, debug: bool = False, trace: bool = True, @@ -181,10 +182,9 @@ def _append_component( :param status: The new component status :return: The updated components dictionary """ - # Determine the component key and purl based on component type if id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: - purl = new_component['purl'][0] # Take first purl for these component types + purl = new_component['purl'][0] # Take the first purl for these component types else: purl = new_component['purl'] @@ -195,14 +195,13 @@ def _append_component( 'licenses': {}, 'status': status, } - if not new_component.get('licenses'): - self.print_stderr(f'WARNING: Results missing licenses. Skipping.') + self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}') return components # Process licenses for this component - for l in new_component['licenses']: - if l.get('name'): - spdxid = l['name'] + for license_item in new_component['licenses']: + if license_item.get('name'): + spdxid = license_item['name'] components[component_key]['licenses'][spdxid] = { 'spdxid': spdxid, 'copyleft': self.license_util.is_copyleft(spdxid), @@ -210,71 +209,103 @@ def _append_component( } return components - def _get_components_from_results(self, results: Dict[str, Any]) -> list or None: + def _get_components_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: """ - Process the results dictionary to extract and format component information. - - This function iterates through the results dictionary, identifying components from - different sources (files, snippets, and dependencies). It consolidates this information - into a list of unique components, each with its associated licenses and other details. + Extract and process file and snippet components from results. :param results: A dictionary containing the raw results of a component scan - :return: A list of dictionaries, each representing a unique component with its details + :param components: Existing components dictionary to update + :return: Updated components dictionary with file and snippet data """ - if results is None: - self.print_stderr(f'ERROR: Results cannot be empty') - return None - components = {} for component in results.values(): for c in component: component_id = c.get('id') if not component_id: - self.print_stderr(f'WARNING: Result missing id. Skipping.') + self.print_debug(f'WARNING: Result missing id. Skipping: {c}') continue status = c.get('status') - if not component_id: - self.print_stderr(f'WARNING: Result missing status. Skipping.') + if not status: + self.print_debug(f'WARNING: Result missing status. Skipping: {c}') continue if component_id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: if not c.get('purl'): - self.print_stderr(f'WARNING: Result missing purl. Skipping.') + self.print_debug(f'WARNING: Result missing purl. Skipping: {c}') continue if len(c.get('purl')) <= 0: - self.print_stderr(f'WARNING: Result missing purls. Skipping.') + self.print_debug(f'WARNING: Result missing purls. Skipping: {c}') continue if not c.get('version'): - self.print_stderr(f'WARNING: Result missing version. Skipping.') + self.print_msg(f'WARNING: Result missing version. Skipping: {c}') continue component_key = f'{c["purl"][0]}@{c["version"]}' - # Initialize or update the component entry if component_key not in components: components = self._append_component(components, c, component_id, status) + # End component loop + # End components loop + return components - if c['id'] == ComponentID.DEPENDENCY.value: + def _get_dependencies_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract and process dependency components from results. + + :param results: A dictionary containing the raw results of a component scan + :param components: Existing components dictionary to update + :return: Updated components dictionary with dependency data + """ + for component in results.values(): + for c in component: + component_id = c.get('id') + if not component_id: + self.print_debug(f'WARNING: Result missing id. Skipping: {c}') + continue + status = c.get('status') + if not status: + self.print_debug(f'WARNING: Result missing status. Skipping: {c}') + continue + if component_id == ComponentID.DEPENDENCY.value: if c.get('dependencies') is None: continue - for d in c['dependencies']: - if not d.get('purl'): - self.print_stderr(f'WARNING: Result missing purl. Skipping.') - continue - if len(d.get('purl')) <= 0: - self.print_stderr(f'WARNING: Result missing purls. Skipping.') + for dependency in c['dependencies']: + if not dependency.get('purl'): + self.print_debug(f'WARNING: Dependency result missing purl. Skipping: {dependency}') continue - if not d.get('version'): - self.print_stderr(f'WARNING: Result missing version. Skipping.') + if not dependency.get('version'): + self.print_msg(f'WARNING: Dependency result missing version. Skipping: {dependency}') continue - component_key = f'{d["purl"]}@{d["version"]}' + component_key = f'{dependency["purl"]}@{dependency["version"]}' if component_key not in components: - components = self._append_component(components, d, component_id, status) - # End of dependencies loop - # End if - # End of component loop - # End of results loop - results = list(components.values()) - for component in results: + components = self._append_component(components, dependency, component_id, status) + # End dependency loop + # End component loop + # End of result loop + return components + + def _get_components_from_results(self, results: Dict[str, Any]) -> list or None: + """ + Process the results dictionary to extract and format component information. + + This function iterates through the results dictionary, identifying components from + different sources (files, snippets, and dependencies). It consolidates this information + into a list of unique components, each with its associated licenses and other details. + + :param results: A dictionary containing the raw results of a component scan + :return: A list of dictionaries, each representing a unique component with its details + """ + if results is None: + self.print_stderr('ERROR: Results cannot be empty') + return None + + components = {} + # Extract file and snippet components + components = self._get_components_data(results, components) + # Extract dependency components + components = self._get_dependencies_data(results, components) + # Convert to list and process licenses + results_list = list(components.values()) + for component in results_list: component['licenses'] = list(component['licenses'].values()) - return results + return results_list def generate_table(self, headers, rows, centered_columns=None): """ @@ -403,7 +434,6 @@ def _get_components(self): components = self._get_components_from_results(self.results) return components - # # End of PolicyCheck Class # From 63cedfe58e1a9c83ecff236d514ed1767b85a9a2 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Tue, 10 Jun 2025 08:58:45 +0100 Subject: [PATCH 340/489] reformatted change details --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e21f3c1f..838c83b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -### [1.25.0] - 2025-06-02 +## [1.25.0] - 2025-06-10 ### Added - Add `fh2` hash while fingerprinting mixed line ending files +### Modified +- Updated `inspect` debug/warning statements ## [1.24.0] - 2025-05-28 ### Added From 7e19648ab92bef7ea49fb990b5a0c28069154eb0 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Thu, 12 Jun 2025 09:39:31 -0300 Subject: [PATCH 341/489] Bug fix: Exclude dependency components from undeclared components list * bug:SP-2714 Removes dependency components from undeclared components list * chore:Updates CHANGELOG.md file * chore:Updates version to v1.25.1 --- CHANGELOG.md | 5 ++ src/scanoss/__init__.py | 2 +- src/scanoss/inspection/copyleft.py | 29 ++++++- src/scanoss/inspection/policy_check.py | 77 +++++++------------ .../inspection/undeclared_component.py | 64 +++++++++++---- tests/test_policy_inspect.py | 56 +++----------- 6 files changed, 119 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 838c83b5..fc1d65c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.25.1] - 2025-06-12 +### Fixed +- Removed dependency components from the undeclared component list in the `inspect` subcommand. + ## [1.25.0] - 2025-06-10 ### Added - Add `fh2` hash while fingerprinting mixed line ending files @@ -530,3 +534,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.23.0]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.23.0 [1.24.0]: https://github.com/scanoss/scanoss.py/compare/v1.23.0...v1.24.0 [1.25.0]: https://github.com/scanoss/scanoss.py/compare/v1.24.0...v1.25.0 +[1.25.1]: https://github.com/scanoss/scanoss.py/compare/v1.25.0...v1.25.1 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 629fead7..4f23f227 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.25.0' +__version__ = '1.25.1' diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index ebee8211..81aee21c 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -23,7 +23,8 @@ """ import json -from typing import Dict, Any +from typing import Any, Dict + from .policy_check import PolicyCheck, PolicyStatus @@ -33,7 +34,7 @@ class Copyleft(PolicyCheck): Inspects components for copyleft licenses """ - def __init__( + def __init__( # noqa: PLR0913 self, debug: bool = False, trace: bool = True, @@ -158,6 +159,30 @@ def _filter_components_with_copyleft_licenses(self, components: list) -> list: self.print_debug(f'Copyleft components: {filtered_components}') return filtered_components + def _get_components(self): + """ + Extract and process components from results and their dependencies. + + This method performs the following steps: + 1. Validates that `self.results` is loaded. Returns `None` if not. + 2. Extracts file, snippet, and dependency components into a dictionary. + 3. Converts components to a list and processes their licenses. + + :return: A list of processed components with license data, or `None` if `self.results` is not set. + """ + if self.results is None: + return None + + components: dict = {} + # Extract component and license data from file and dependency results. Both helpers mutate `components` + self._get_components_data(self.results, components) + self._get_dependencies_data(self.results, components) + # Convert to list and process licenses + results_list = list(components.values()) + for component in results_list: + component['licenses'] = list(component['licenses'].values()) + return results_list + def run(self): """ Run the copyleft license inspection process. diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index e60b2974..c1e2f4cb 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -166,6 +166,30 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: """ pass + @abstractmethod + def _get_components(self): + """ + Retrieve and process components from the preloaded results. + + This method performs the following steps: + 1. Checks if the results have been previously loaded (self.results). + 2. Extracts and processes components from the loaded results. + + :return: A list of processed components, or None if an error occurred during any step. + + Possible reasons for returning None include: + - Results not loaded (self.results is None) + - Failure to extract components from the results + + Note: + - This method assumes that the results have been previously loaded and stored in self.results. + - Implementations must extract components (e.g. via `_get_components_data`, + `_get_dependencies_data`, or other helpers). + - If `self.results` is `None`, simply return `None`. + """ + pass + + def _append_component( self, components: Dict[str, Any], new_component: Dict[str, Any], id: str, status: str ) -> Dict[str, Any]: @@ -223,6 +247,9 @@ def _get_components_data(self, results: Dict[str, Any], components: Dict[str, An if not component_id: self.print_debug(f'WARNING: Result missing id. Skipping: {c}') continue + ## Skip dependency + if component_id == ComponentID.DEPENDENCY.value: + continue status = c.get('status') if not status: self.print_debug(f'WARNING: Result missing status. Skipping: {c}') @@ -280,33 +307,6 @@ def _get_dependencies_data(self, results: Dict[str, Any], components: Dict[str, # End of result loop return components - def _get_components_from_results(self, results: Dict[str, Any]) -> list or None: - """ - Process the results dictionary to extract and format component information. - - This function iterates through the results dictionary, identifying components from - different sources (files, snippets, and dependencies). It consolidates this information - into a list of unique components, each with its associated licenses and other details. - - :param results: A dictionary containing the raw results of a component scan - :return: A list of dictionaries, each representing a unique component with its details - """ - if results is None: - self.print_stderr('ERROR: Results cannot be empty') - return None - - components = {} - # Extract file and snippet components - components = self._get_components_data(results, components) - # Extract dependency components - components = self._get_dependencies_data(results, components) - # Convert to list and process licenses - results_list = list(components.values()) - for component in results_list: - component['licenses'] = list(component['licenses'].values()) - - return results_list - def generate_table(self, headers, rows, centered_columns=None): """ Generate a Markdown table. @@ -411,29 +411,6 @@ def _load_input_file(self): self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return None - def _get_components(self): - """ - Retrieve and process components from the preloaded results. - - This method performs the following steps: - 1. Checks if the results have been previously loaded (self.results). - 2. Extracts and processes components from the loaded results. - - :return: A list of processed components, or None if an error occurred during any step. - Possible reasons for returning None include: - - Results not loaded (self.results is None) - - Failure to extract components from the results - - Note: - - This method assumes that the results have been previously loaded and stored in self.results. - - If results is None, the method returns None without performing any further operations. - - The actual processing of components is delegated to the _get_components_from_results method. - """ - if self.results is None: - return None - components = self._get_components_from_results(self.results) - return components - # # End of PolicyCheck Class # diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index 5b222406..865f2769 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -23,7 +23,8 @@ """ import json -from typing import Dict, Any +from typing import Any, Dict + from .policy_check import PolicyCheck, PolicyStatus @@ -33,7 +34,7 @@ class UndeclaredComponent(PolicyCheck): Inspects for undeclared components """ - def __init__( + def __init__( # noqa: PLR0913 self, debug: bool = False, trace: bool = True, @@ -73,7 +74,7 @@ def _get_undeclared_component(self, components: list) -> list or None: :return: List of undeclared components """ if components is None: - self.print_debug(f'WARNING: No components provided!') + self.print_debug('WARNING: No components provided!') return None undeclared_components = [] for component in components: @@ -87,25 +88,35 @@ def _get_jira_summary(self, components: list) -> str: """ Get a summary of the undeclared components. + :param components: List of all components + :return: Component summary markdown + """ + + """ + Get a summary of the undeclared components. + :param components: List of all components :return: Component summary markdown """ if len(components) > 0: + json_content = json.dumps(self._generate_scanoss_file(components), indent=2) + if self.sbom_format == 'settings': - json_str = ( - json.dumps(self._generate_scanoss_file(components), indent=2) - .replace('\n', '\\n') - .replace('"', '\\"') + return ( + f'{len(components)} undeclared component(s) were found.\n' + f'Add the following snippet into your `scanoss.json` file\n' + f'{{code:json}}\n' + f'{json_content}\n' + f'{{code}}\n' ) - return f'{len(components)} undeclared component(s) were found.\nAdd the following snippet into your `scanoss.json` file\n{{code:json}}\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n{{code}}\n' else: - json_str = ( - json.dumps(self._generate_scanoss_file(components), indent=2) - .replace('\n', '\\n') - .replace('"', '\\"') + return ( + f'{len(components)} undeclared component(s) were found.\n' + f'Add the following snippet into your `sbom.json` file\n' + f'{{code:json}}\n' + f'{json_content}\n' + f'{{code}}\n' ) - return f'{len(components)} undeclared component(s) were found.\nAdd the following snippet into your `sbom.json` file\n{{code:json}}\n{json.dumps(self._generate_scanoss_file(components), indent=2)}\n{{code}}\n' - return f'{len(components)} undeclared component(s) were found.\\n' def _get_summary(self, components: list) -> str: @@ -190,7 +201,7 @@ def _get_unique_components(self, components: list) -> list: """ unique_components = {} if components is None: - self.print_stderr(f'WARNING: No components provided!') + self.print_stderr('WARNING: No components provided!') return [] for component in components: @@ -225,6 +236,29 @@ def _generate_sbom_file(self, components: list) -> dict: return sbom + def _get_components(self): + """ + Extract and process components from file results only. + + This method performs the following steps: + 1. Validates if `self.results` is loaded. Returns `None` if not loaded. + 2. Extracts file and snippet components into a dictionary. + 3. Converts the components dictionary into a list of components. + 4. Processes the licenses for each component by converting them into a list. + + :return: A list of processed components with their licenses, or `None` if `self.results` is not set. + """ + if self.results is None: + return None + components: dict = {} + # Extract file and snippet components + components = self._get_components_data(self.results, components) + # Convert to list and process licenses + results_list = list(components.values()) + for component in results_list: + component['licenses'] = list(component['licenses'].values()) + return results_list + def run(self): """ Run the undeclared component inspection process. diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index d39e0cdf..50792a47 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -179,7 +179,7 @@ def test_undeclared_policy(self): status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] - expected_summary_output = """5 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json { @@ -189,17 +189,11 @@ def test_undeclared_policy(self): }, { "purl": "pkg:github/scanoss/wfp" - }, - { - "purl": "pkg:npm/%40electron/rebuild" - }, - { - "purl": "pkg:npm/%40emotion/react" } ] }``` """ - self.assertEqual(len(details['components']), 5) + self.assertEqual(len(details['components']), 3) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) ) @@ -222,11 +216,9 @@ def test_undeclared_policy_markdown(self): | - | - | - | | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | - | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | - | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | - | pkg:npm/%40emotion/react | 11.13.3 | MIT | """ + | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | """ - expected_summary_output = """5 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json { @@ -236,13 +228,7 @@ def test_undeclared_policy_markdown(self): }, { "purl": "pkg:github/scanoss/wfp" - }, - { - "purl": "pkg:npm/%40electron/rebuild" - }, - { - "purl": "pkg:npm/%40emotion/react" - } + } ] }``` """ @@ -273,11 +259,9 @@ def test_undeclared_policy_markdown_scanoss_summary(self): | - | - | - | | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | - | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | - | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | - | pkg:npm/%40emotion/react | 11.13.3 | MIT | """ + | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | """ - expected_summary_output = """5 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file ```json @@ -289,12 +273,6 @@ def test_undeclared_policy_markdown_scanoss_summary(self): }, { "purl": "pkg:github/scanoss/wfp" - }, - { - "purl": "pkg:npm/%40electron/rebuild" - }, - { - "purl": "pkg:npm/%40emotion/react" } ] } @@ -322,7 +300,7 @@ def test_undeclared_policy_scanoss_summary(self): status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] - expected_summary_output = """5 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file ```json @@ -334,19 +312,13 @@ def test_undeclared_policy_scanoss_summary(self): }, { "purl": "pkg:github/scanoss/wfp" - }, - { - "purl": "pkg:npm/%40electron/rebuild" - }, - { - "purl": "pkg:npm/%40emotion/react" } ] } } ```""" self.assertEqual(status, 0) - self.assertEqual(len(details['components']), 5) + self.assertEqual(len(details['components']), 3) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) ) @@ -363,10 +335,8 @@ def test_undeclared_policy_jira_markdown_output(self): |pkg:github/scanoss/scanner.c|1.3.3|BSD-2-Clause - GPL-2.0-only| |pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only| |pkg:github/scanoss/wfp|6afc1f6|Zlib - GPL-2.0-only| -|pkg:npm/%40electron/rebuild|3.7.0|MIT| -|pkg:npm/%40emotion/react|11.13.3|MIT| """ - expected_summary_output = """5 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file {code:json} { @@ -377,12 +347,6 @@ def test_undeclared_policy_jira_markdown_output(self): }, { "purl": "pkg:github/scanoss/wfp" - }, - { - "purl": "pkg:npm/%40electron/rebuild" - }, - { - "purl": "pkg:npm/%40emotion/react" } ] } From 02f00925247b4d2d97d8e2a1191c7a4b13bb0644 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:03:59 -0300 Subject: [PATCH 342/489] Bug/inspect subcommand * bug:SP-2745 Avoids errors on inspect subcommand when no version is declared on results * chore:Upgrades version to 1.25.2 * chore:Updates CHANGELOG.md file * chore:SP-2746 Prioritizes licenses by source in inspect copyleft subcommand --- CHANGELOG.md | 9 +- src/scanoss/__init__.py | 2 +- src/scanoss/inspection/copyleft.py | 6 +- src/scanoss/inspection/policy_check.py | 88 +++++++++++++++++-- .../inspection/undeclared_component.py | 5 +- tests/test_policy_inspect.py | 17 ++-- 6 files changed, 98 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc1d65c9..456a5b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.25.2] - 2025-06-18 +### Fixed +- Fixed errors when no versions are declared in scanner results for `inspect` subcommand +### Changed +- Prioritized licenses by source priority in `inspect copyleft` subcommand + ## [1.25.1] - 2025-06-12 ### Fixed - Removed dependency components from the undeclared component list in the `inspect` subcommand. @@ -534,4 +540,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.23.0]: https://github.com/scanoss/scanoss.py/compare/v1.22.0...v1.23.0 [1.24.0]: https://github.com/scanoss/scanoss.py/compare/v1.23.0...v1.24.0 [1.25.0]: https://github.com/scanoss/scanoss.py/compare/v1.24.0...v1.25.0 -[1.25.1]: https://github.com/scanoss/scanoss.py/compare/v1.25.0...v1.25.1 \ No newline at end of file +[1.25.1]: https://github.com/scanoss/scanoss.py/compare/v1.25.0...v1.25.1 +[1.25.2]: https://github.com/scanoss/scanoss.py/compare/v1.25.1...v1.25.2 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 4f23f227..dc954aac 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.25.1' +__version__ = '1.25.2' diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index 81aee21c..edc17036 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -177,11 +177,7 @@ def _get_components(self): # Extract component and license data from file and dependency results. Both helpers mutate `components` self._get_components_data(self.results, components) self._get_dependencies_data(self.results, components) - # Convert to list and process licenses - results_list = list(components.values()) - for component in results_list: - component['licenses'] = list(component['licenses'].values()) - return results_list + return self._convert_components_to_list(components) def run(self): """ diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index c1e2f4cb..6c5d057d 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -212,6 +212,10 @@ def _append_component( else: purl = new_component['purl'] + if not purl: + self.print_debug(f'WARNING: _append_component: No purl found for new component: {new_component}') + return components + component_key = f'{purl}@{new_component["version"]}' components[component_key] = { 'purl': purl, @@ -222,14 +226,21 @@ def _append_component( if not new_component.get('licenses'): self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}') return components + + + licenses_order_by_source_priority = self._get_licenses_order_by_source_priority(new_component['licenses']) # Process licenses for this component - for license_item in new_component['licenses']: + for license_item in licenses_order_by_source_priority: if license_item.get('name'): spdxid = license_item['name'] + source = license_item.get('source') + if not source: + source = 'unknown' components[component_key]['licenses'][spdxid] = { 'spdxid': spdxid, 'copyleft': self.license_util.is_copyleft(spdxid), 'url': self.license_util.get_spdx_url(spdxid), + 'source': source, } return components @@ -261,10 +272,12 @@ def _get_components_data(self, results: Dict[str, Any], components: Dict[str, An if len(c.get('purl')) <= 0: self.print_debug(f'WARNING: Result missing purls. Skipping: {c}') continue - if not c.get('version'): - self.print_msg(f'WARNING: Result missing version. Skipping: {c}') - continue - component_key = f'{c["purl"][0]}@{c["version"]}' + version = c.get('version') + if not version: + self.print_debug(f'WARNING: Result missing version. Setting it to unknown: {c}') + version = 'unknown' + c['version'] = version #If no version exists. Set 'unknown' version to current component + component_key = f'{c["purl"][0]}@{version}' if component_key not in components: components = self._append_component(components, c, component_id, status) # End component loop @@ -296,10 +309,12 @@ def _get_dependencies_data(self, results: Dict[str, Any], components: Dict[str, if not dependency.get('purl'): self.print_debug(f'WARNING: Dependency result missing purl. Skipping: {dependency}') continue - if not dependency.get('version'): - self.print_msg(f'WARNING: Dependency result missing version. Skipping: {dependency}') - continue - component_key = f'{dependency["purl"]}@{dependency["version"]}' + version = c.get('version') + if not version: + self.print_debug(f'WARNING: Result missing version. Setting it to unknown: {c}') + version = 'unknown' + c['version'] = version # If no version exists. Set 'unknown' version to current component + component_key = f'{dependency["purl"]}@{version}' if component_key not in components: components = self._append_component(components, dependency, component_id, status) # End dependency loop @@ -411,6 +426,61 @@ def _load_input_file(self): self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return None + def _convert_components_to_list(self, components: dict): + if components is None: + self.print_debug(f'WARNING: Components is empty {self.results}') + return None + results_list = list(components.values()) + for component in results_list: + licenses = component.get('licenses') + if licenses is not None: + component['licenses'] = list(licenses.values()) + else: + self.print_debug(f'WARNING: Licenses missing for: {component}') + component['licenses'] = [] + return results_list + + def _get_licenses_order_by_source_priority(self,licenses_data): + """ + Select licenses based on source priority: + 1. component_declared (highest priority) + 2. license_file + 3. file_header + 4. scancode (lowest priority) + + If any high-priority source is found, return only licenses from that source. + If none found, return all licenses. + + Returns: list with ordered licenses by source. + """ + # Define priority order (highest to lowest) + priority_sources = ['component_declared', 'license_file', 'file_header', 'scancode'] + + # Group licenses by source + licenses_by_source = {} + for license_item in licenses_data: + + source = license_item.get('source', 'unknown') + if source not in licenses_by_source: + licenses_by_source[source] = {} + + license_name = license_item.get('name') + if license_name: + # Use license name as key, store full license object as value + # If duplicate license names exist in same source, the last one wins + licenses_by_source[source][license_name] = license_item + + # Find the highest priority source that has licenses + for priority_source in priority_sources: + if priority_source in licenses_by_source: + self.print_trace(f'Choosing {priority_source} as source') + return list(licenses_by_source[priority_source].values()) + + # If no priority sources found, combine all licenses into a single list + self.print_debug("No priority sources found, returning all licenses as list") + return licenses_data + + # # End of PolicyCheck Class # diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index 865f2769..f0c174fd 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -254,10 +254,7 @@ def _get_components(self): # Extract file and snippet components components = self._get_components_data(self.results, components) # Convert to list and process licenses - results_list = list(components.values()) - for component in results_list: - component['licenses'] = list(component['licenses'].values()) - return results_list + return self._convert_components_to_list(components) def run(self): """ diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 50792a47..771a7aa7 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -114,7 +114,7 @@ def test_copyleft_policy_explicit(self): copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='MIT') status, results = copyleft.run() details = json.loads(results['details']) - self.assertEqual(len(details['components']), 3) + self.assertEqual(len(details['components']), 2) self.assertEqual(status, 0) """ @@ -144,11 +144,10 @@ def test_copyleft_policy_markdown(self): expected_detail_output = ( '### Copyleft licenses \n | Component | Version | License | URL | Copyleft |\n' ' | - | :-: | - | - | :-: |\n' - '| pkg:github/scanoss/engine | 4.0.4 | MIT | https://spdx.org/licenses/MIT.html | YES | \n' ' | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | https://spdx.org/licenses/MIT.html | YES |\n' '| pkg:npm/%40emotion/react | 11.13.3 | MIT | https://spdx.org/licenses/MIT.html | YES | \n' ) - expected_summary_output = '3 component(s) with copyleft licenses were found.\n' + expected_summary_output = '2 component(s) with copyleft licenses were found.\n' self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', results['details']), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output), @@ -214,9 +213,9 @@ def test_undeclared_policy_markdown(self): expected_details_output = """ ### Undeclared components | Component | Version | License | | - | - | - | - | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | + | pkg:github/scanoss/scanner.c | 1.3.3 | GPL-2.0-only | | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | - | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | """ + | pkg:github/scanoss/wfp | 6afc1f6 | GPL-2.0-only | """ expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `sbom.json` file @@ -257,9 +256,9 @@ def test_undeclared_policy_markdown_scanoss_summary(self): expected_details_output = """ ### Undeclared components | Component | Version | License | | - | - | - | - | pkg:github/scanoss/scanner.c | 1.3.3 | BSD-2-Clause - GPL-2.0-only | + | pkg:github/scanoss/scanner.c | 1.3.3 | GPL-2.0-only | | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | - | pkg:github/scanoss/wfp | 6afc1f6 | Zlib - GPL-2.0-only | """ + | pkg:github/scanoss/wfp | 6afc1f6 | GPL-2.0-only | """ expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file @@ -332,9 +331,9 @@ def test_undeclared_policy_jira_markdown_output(self): details = results['details'] summary = results['summary'] expected_details_output = """|*Component*|*Version*|*License*| -|pkg:github/scanoss/scanner.c|1.3.3|BSD-2-Clause - GPL-2.0-only| +|pkg:github/scanoss/scanner.c|1.3.3|GPL-2.0-only| |pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only| -|pkg:github/scanoss/wfp|6afc1f6|Zlib - GPL-2.0-only| +|pkg:github/scanoss/wfp|6afc1f6|GPL-2.0-only| """ expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file From 9f4b28ebb35eb1d90fd7abe18cb66d364f8545b6 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Fri, 20 Jun 2025 11:27:13 -0300 Subject: [PATCH 343/489] feat: Implement inspect license-summary and component-summary commands * feat:SP-2753 Implements inspect license-summary subcommand * feat:SP-2754 Implements inspect component summary subcommand * chore:SP-2756 Adds examples for inspect summary on CLIENT_HELP.md * chore:SP-2755 Updates CHANGELOG.md file * chore: Upgrades version to v1.26.0 * bug:SP-2765 Keeps the same component in 'pending' status when at least one instance of that component with 'pending' status is found * chore:SP-2768 Refactor on inspect undeclared format md view --- CHANGELOG.md | 13 +- CLIENT_HELP.md | 44 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 135 ++++++- src/scanoss/inspection/__init__.py | 2 +- src/scanoss/inspection/component_summary.py | 85 ++++ src/scanoss/inspection/copyleft.py | 10 +- src/scanoss/inspection/inspect_base.py | 378 ++++++++++++++++++ src/scanoss/inspection/license_summary.py | 163 ++++++++ src/scanoss/inspection/policy_check.py | 289 +------------ .../inspection/undeclared_component.py | 63 ++- tests/data/empty-result.json | 1 + tests/test_policy_inspect.py | 83 ++-- 13 files changed, 934 insertions(+), 334 deletions(-) create mode 100644 src/scanoss/inspection/component_summary.py create mode 100644 src/scanoss/inspection/inspect_base.py create mode 100644 src/scanoss/inspection/license_summary.py create mode 100644 tests/data/empty-result.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 456a5b4a..ba5f9eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.26.0] - 2025-06-20 +### Added +- New `inspect license-summary` subcommand to generate license summaries from scan results +- New `inspect component-summary` subcommand to generate component summaries from scan results +- Added examples for inspect summary commands in CLIENT_HELP.md +### Fixed +- Fixed `inspect undeclared` bug where pending status now takes precedence over identified status for the same component +### Changed +- Modified `inspect undeclared` view for `md` format to remove version information + ## [1.25.2] - 2025-06-18 ### Fixed - Fixed errors when no versions are declared in scanner results for `inspect` subcommand @@ -541,4 +551,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.24.0]: https://github.com/scanoss/scanoss.py/compare/v1.23.0...v1.24.0 [1.25.0]: https://github.com/scanoss/scanoss.py/compare/v1.24.0...v1.25.0 [1.25.1]: https://github.com/scanoss/scanoss.py/compare/v1.25.0...v1.25.1 -[1.25.2]: https://github.com/scanoss/scanoss.py/compare/v1.25.1...v1.25.2 \ No newline at end of file +[1.25.2]: https://github.com/scanoss/scanoss.py/compare/v1.25.1...v1.25.2 +[1.26.0]: https://github.com/scanoss/scanoss.py/compare/v1.25.2...v1.26.0 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 34e9d824..16e6aef0 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -369,6 +369,8 @@ Details, such as license compliance or component declarations, can be examined. For example: * Copyleft (`copylefet`) * Undeclared Components (`undeclared`) +* License Summary (`license-summary`) +* Component Summary (`component-summary`) For the latest list of sub-commands, please run: ```bash @@ -434,6 +436,46 @@ The following command can be used to inspect for undeclared components and save scanoss-py insp copyleft -i scan-results.json --output copyleft-summary.jiramd --status copyleft-status.jiramd --format jira_md ``` +#### Inspect for license summary from scan results +The following command can be used to get a license summary from scan results. +```bash +scanoss-py insp license-summary -i scan-results.json --output license-summary.json +``` + +Example with an output file: + +```bash +scanoss-py insp license-summary -i scan-results.json --output license-summary.txt +``` + +#### Inspect for license summary from scan results with custom copyleft licenses +The following command can be used to get a license summary from scan results. + +Example including a license to the default list +```bash +scanoss-py insp license-summary -i scan-results.json --output license-summary.json --include Zlib,MIT +``` + +Example excluding a license from the default list +```bash +scanoss-py insp license-summary -i scan-results.json --output license-summary.txt --exclude GPL-2.0-only +``` + +Example getting only explicit declared licenses +```bash +scanoss-py insp license-summary -i scan-results.json --output license-summary.json --explicit Zlib +``` + +#### Inspect for component summary from scan results +The following command can be used to get a component summary from scan results and save the output. +```bash +scanoss-py insp component-summary -i scan-results.json +``` +Example with an output file: +```bash +scanoss-py insp component-summary -i scan-results.json --output component-summary.json +``` + ### Folder-Scan a Project Folder The new `folder-scan` subcommand performs a comprehensive scan on an entire directory by recursively processing files to generate folder-level fingerprints. It computes CRC64 hashes and simhash values to detect directory-level similarities, which is especially useful for comparing large code bases or detecting duplicate folder structures. @@ -450,4 +492,4 @@ The `container-scan` subcommand allows you to scan Docker container images for d **Usage:** ```shell scanoss-py container-scan image_name:tag -o container-scan-results.json -``` +``` \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index dc954aac..a252dab9 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.25.2' +__version__ = '1.26.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ec88177a..418a68e8 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -32,6 +32,8 @@ import pypac from scanoss.cryptography import Cryptography, create_cryptography_config_from_args +from scanoss.inspection.component_summary import ComponentSummary +from scanoss.inspection.license_summary import LicenseSummary from scanoss.scanners.container_scanner import ( DEFAULT_SYFT_COMMAND, DEFAULT_SYFT_TIMEOUT, @@ -531,6 +533,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) p_results.set_defaults(func=results) + ########################################### INSPECT SUBCOMMAND ########################################### # Sub-command: inspect p_inspect = subparsers.add_parser( 'inspect', aliases=['insp', 'ins'], description=f'Inspect results: {__version__}', help='Inspect results' @@ -539,24 +542,26 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_inspect_sub = p_inspect.add_subparsers( title='Inspect Commands', dest='subparsercmd', description='Inspect sub-commands', help='Inspect sub-commands' ) + + ####### INSPECT: Copyleft ###### # Inspect Sub-command: inspect copyleft p_copyleft = p_inspect_sub.add_parser( 'copyleft', aliases=['cp'], description='Inspect for copyleft licenses', help='Inspect for copyleft licenses' ) - p_copyleft.add_argument( - '--include', - help='List of Copyleft licenses to append to the default list. Provide licenses as a comma-separated list.', - ) - p_copyleft.add_argument( - '--exclude', - help='List of Copyleft licenses to remove from default list. Provide licenses as a comma-separated list.', + + ####### INSPECT: License Summary ###### + # Inspect Sub-command: inspect license summary + p_license_summary = p_inspect_sub.add_parser( + 'license-summary', aliases=['lic-summary', 'licsum'], description='Get license summary', + help='Get detected license summary from scan results' ) - p_copyleft.add_argument( - '--explicit', - help='Explicit list of Copyleft licenses to consider. Provide licenses as a comma-separated list.s', + + p_component_summary = p_inspect_sub.add_parser( + 'component-summary', aliases=['comp-summary', 'compsum'], description='Get component summary', + help='Get detected component summary from scan results' ) - p_copyleft.set_defaults(func=inspect_copyleft) + ####### INSPECT: Undeclared components ###### # Inspect Sub-command: inspect undeclared p_undeclared = p_inspect_sub.add_parser( 'undeclared', @@ -571,7 +576,33 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 default='settings', help='Sbom format for status output', ) + + # Add common commands for inspect copyleft and license summary + for p in [p_copyleft, p_license_summary]: + p.add_argument( + '--include', + help='List of Copyleft licenses to append to the default list. Provide licenses as a comma-separated list.', + ) + p.add_argument( + '--exclude', + help='List of Copyleft licenses to remove from default list. Provide licenses as a comma-separated list.', + ) + p.add_argument( + '--explicit', + help='Explicit list of Copyleft licenses to consider. Provide licenses as a comma-separated list.s', + ) + + # Add common commands for inspect copyleft and license summary + for p in [p_license_summary, p_component_summary]: + p.add_argument('-i', '--input', nargs='?', help='Path to results file') + p.add_argument('-o', '--output', type=str, help='Save summary into a file') + p_undeclared.set_defaults(func=inspect_undeclared) + p_copyleft.set_defaults(func=inspect_copyleft) + p_license_summary.set_defaults(func=inspect_license_summary) + p_component_summary.set_defaults(func=inspect_component_summary) + + ########################################### END INSPECT SUBCOMMAND ########################################### # Sub-command: folder-scan p_folder_scan = subparsers.add_parser( @@ -825,6 +856,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_results, p_undeclared, p_copyleft, + p_license_summary, + p_component_summary, c_provenance, p_folder_scan, p_folder_hash, @@ -1279,7 +1312,7 @@ def convert(parser, args): if not success: sys.exit(1) - +################################ INSPECT handlers ################################ def inspect_copyleft(parser, args): """ Run the "inspect" sub-command @@ -1356,13 +1389,73 @@ def inspect_undeclared(parser, args): status, _ = i_undeclared.run() sys.exit(status) +def inspect_license_summary(parser, args): + """ + Run the "inspect" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if args.input is None: + print_stderr('Please specify an input file to inspect') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + output: str = None + if args.output: + output = args.output + open(output, 'w').close() + + i_license_summary = LicenseSummary( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + filepath=args.input, + output=output, + include=args.include, + exclude=args.exclude, + explicit=args.explicit, + ) + i_license_summary.run() + +def inspect_component_summary(parser, args): + """ + Run the "inspect" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if args.input is None: + print_stderr('Please specify an input file to inspect') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + output: str = None + if args.output: + output = args.output + open(output, 'w').close() + + i_component_summary = ComponentSummary( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + filepath=args.input, + output=output, + ) + i_component_summary.run() + +################################ End inspect handlers ################################ def utils_certloc(*_): """ Run the "utils certloc" sub-command :param _: ignored/unused """ - import certifi + import certifi # noqa: PLC0415,I001 print(f'CA Cert File: {certifi.where()}') @@ -1373,11 +1466,11 @@ def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912 :param _: ignore/unused :param args: Parsed arguments """ - import socket - import traceback - from urllib.parse import urlparse + import socket # noqa: PLC0415,I001 + import traceback # noqa: PLC0415,I001 + from urllib.parse import urlparse # noqa: PLC0415,I001 - from OpenSSL import SSL, crypto + from OpenSSL import SSL, crypto # noqa: PLC0415,I001 file = sys.stdout if args.output: @@ -1425,7 +1518,7 @@ def utils_pac_proxy(_, args): :param _: ignore/unused :param args: Parsed arguments """ - from pypac.resolver import ProxyResolver + from pypac.resolver import ProxyResolver # noqa: PLC0415,I001 if not args.pac: print_stderr('Error: No pac file option specified.') @@ -1499,7 +1592,7 @@ def crypto_algorithms(parser, args): sys.exit(1) except Exception as e: if args.debug: - import traceback + import traceback # noqa: PLC0415,I001 traceback.print_exc() print_stderr(f'ERROR: {e}') @@ -1541,7 +1634,7 @@ def crypto_hints(parser, args): sys.exit(1) except Exception as e: if args.debug: - import traceback + import traceback # noqa: PLC0415,I001 traceback.print_exc() print_stderr(f'ERROR: {e}') @@ -1583,7 +1676,7 @@ def crypto_versions_in_range(parser, args): sys.exit(1) except Exception as e: if args.debug: - import traceback + import traceback # noqa: PLC0415,I001 traceback.print_exc() print_stderr(f'ERROR: {e}') diff --git a/src/scanoss/inspection/__init__.py b/src/scanoss/inspection/__init__.py index ebd5917f..1e95c46d 100644 --- a/src/scanoss/inspection/__init__.py +++ b/src/scanoss/inspection/__init__.py @@ -1,7 +1,7 @@ """ SPDX-License-Identifier: MIT - Copyright (c) 2024, SCANOSS + Copyright (c) 2025, SCANOSS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/scanoss/inspection/component_summary.py b/src/scanoss/inspection/component_summary.py new file mode 100644 index 00000000..ad25df2c --- /dev/null +++ b/src/scanoss/inspection/component_summary.py @@ -0,0 +1,85 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json + +from .inspect_base import InspectBase + + +class ComponentSummary(InspectBase): + def _get_component_summary_from_components(self, scan_components: list)-> dict: + """ + Get a component summary from detected components. + + :param components: List of all components + :return: Dict with license summary information + """ + components: list = [] + undeclared_components = 0 + total_components = 0 + for component in scan_components: + total_components += component['count'] + undeclared_components += component['undeclared'] + components.append({ + 'purl': component['purl'], + 'version': component['version'], + 'count': component['count'], + 'undeclared': component['undeclared'], + 'declared': component['count'] - component['undeclared'], + }) + ## End for loop components + return { + 'components': components, + 'total': total_components, + 'undeclared': undeclared_components, + 'declared': total_components - undeclared_components, + } + + def _get_components(self): + """ + Extract and process components from results and their dependencies. + + This method performs the following steps: + 1. Validates that `self.results` is loaded. Returns `None` if not. + 2. Extracts file, snippet, and dependency components into a dictionary. + 3. Converts components to a list and processes their licenses. + + :return: A list of processed components with license data, or `None` if `self.results` is not set. + """ + if self.results is None: + return None + + components: dict = {} + # Extract component and license data from file and dependency results. Both helpers mutate `components` + self._get_components_data(self.results, components) + return self._convert_components_to_list(components) + + def run(self): + components = self._get_components() + component_summary = self._get_component_summary_from_components(components) + self.print_to_file_or_stdout(json.dumps(component_summary, indent=2), self.output) + return component_summary +# +# End of ComponentSummary Class +# \ No newline at end of file diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index edc17036..fbef3124 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -1,7 +1,7 @@ """ SPDX-License-Identifier: MIT - Copyright (c) 2024, SCANOSS + Copyright (c) 2025, SCANOSS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -151,7 +151,15 @@ def _filter_components_with_copyleft_licenses(self, components: list) -> list: for component in components: copyleft_licenses = [lic for lic in component['licenses'] if lic['copyleft']] if copyleft_licenses: + # Remove unused keys + del component['count'] + del component['declared'] + del component['undeclared'] filtered_component = component + # Remove 'count' from each license using pop + for lic in copyleft_licenses: + lic.pop('count', None) # None is default value if key doesn't exist + filtered_component['licenses'] = copyleft_licenses del filtered_component['status'] filtered_components.append(filtered_component) diff --git a/src/scanoss/inspection/inspect_base.py b/src/scanoss/inspection/inspect_base.py new file mode 100644 index 00000000..df8c2fa4 --- /dev/null +++ b/src/scanoss/inspection/inspect_base.py @@ -0,0 +1,378 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import os.path +from abc import abstractmethod +from enum import Enum +from typing import Any, Dict + +from ..scanossbase import ScanossBase +from .utils.license_utils import LicenseUtil + + +class ComponentID(Enum): + """ + Enumeration representing different types of software components. + + Attributes: + FILE (str): Represents a file component (value: "file"). + SNIPPET (str): Represents a code snippet component (value: "snippet"). + DEPENDENCY (str): Represents a dependency component (value: "dependency"). + """ + + FILE = 'file' + SNIPPET = 'snippet' + DEPENDENCY = 'dependency' + + +# +# End of ComponentID Class +# + + +class InspectBase(ScanossBase): + """ + A base class to perform inspections over scan results. + + This class provides a basic for scan results inspection, including methods for + processing scan results components and licenses. + + Inherits from: + ScanossBase: A base class providing common functionality for SCANOSS-related operations. + """ + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = True, + quiet: bool = False, + filepath: str = None, + output: str = None, + ): + super().__init__(debug, trace, quiet) + self.license_util = LicenseUtil() + self.filepath = filepath + self.output = output + self.results = self._load_input_file() + + @abstractmethod + def _get_components(self): + """ + Retrieve and process components from the preloaded results. + + This method performs the following steps: + 1. Checks if the results have been previously loaded (self.results). + 2. Extracts and processes components from the loaded results. + + :return: A list of processed components, or None if an error occurred during any step. + + Possible reasons for returning None include: + - Results not loaded (self.results is None) + - Failure to extract components from the results + + Note: + - This method assumes that the results have been previously loaded and stored in self.results. + - Implementations must extract components (e.g. via `_get_components_data`, + `_get_dependencies_data`, or other helpers). + - If `self.results` is `None`, simply return `None`. + """ + pass + + def _append_component(self, components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: + """ + Append a new component to the component dictionary. + + This function creates a new entry in the component dictionary for the given component, + initializing all required counters: + - count: Total occurrences of this component (used by both license and component summaries) + - declared: Number of times this component is marked as 'identified' (used by component summary) + - undeclared: Number of times this component is marked as 'pending' (used by component summary) + + Each component also contains a 'licenses' dictionary where each license entry tracks: + - count: Number of times this license appears for this component (used by license summary) + + Args: + components: The existing dictionary of components + new_component: The new component to be added + Returns: + The updated components dictionary + """ + match_id = new_component.get('id') + # Determine the component key and purl based on component type + if match_id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: + purl = new_component['purl'][0] # Take the first purl for these component types + else: + purl = new_component['purl'] + + if not purl: + self.print_debug(f'WARNING: _append_component: No purl found for new component: {new_component}') + return components + + component_key = f'{purl}@{new_component["version"]}' + status = new_component.get('status') + + if component_key in components: + # Component already exists, update component counters and try to append a new license + self._update_component_counters(components[component_key], status) + self._append_license_to_component(components, new_component, component_key) + # Maintain 'pending' status - takes precedence over 'identified' + if status == 'pending': + components[component_key]['status'] = "pending" + return components + + # Create a new component + components[component_key] = { + 'purl': purl, + 'version': new_component['version'], + 'licenses': {}, + 'status': status, + 'count': 1, + 'declared': 1 if status == 'identified' else 0, + 'undeclared': 1 if status == 'pending' else 0 + } + if not new_component.get('licenses'): + self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}') + return components + + ## Append license to component + self._append_license_to_component(components, new_component, component_key) + return components + + def _append_license_to_component(self, + components: Dict[str, Any], new_component: Dict[str, Any], component_key: str) -> None: + """ + Add or update licenses for an existing component. + + For each license in the component: + - If the license already exists, increments its count + - If it's a new license, adds it with an initial count of 1 + + The license count is used by license_summary to track how many times each license appears + across all components. This count contributes to: + - Total number of licenses in the project + - Number of copyleft licenses when the license is marked as copyleft + + Args: + components: Dictionary containing all components + new_component: Component whose licenses need to be processed + component_key: purl + version of the component to be updated + """ + licenses_order_by_source_priority = self._get_licenses_order_by_source_priority(new_component['licenses']) + # Process licenses for this component + for license_item in licenses_order_by_source_priority: + if license_item.get('name'): + spdxid = license_item['name'] + source = license_item.get('source') + if not source: + source = 'unknown' + + if spdxid in components[component_key]['licenses']: + # If license exists, increment counter + components[component_key]['licenses'][spdxid]['count'] += 1 # Increment counter for license + else: + # If a license doesn't exist, create new entry + components[component_key]['licenses'][spdxid] = { + 'spdxid': spdxid, + 'copyleft': self.license_util.is_copyleft(spdxid), + 'url': self.license_util.get_spdx_url(spdxid), + 'source': source, + 'count': 1, # Set counter to 1 on new license + } + + def _update_component_counters(self, component, status): + """Update component counters based on status.""" + component['count'] += 1 + if status == 'identified': + component['declared'] += 1 + else: + component['undeclared'] += 1 + + def _get_components_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract and process file and snippet components from results. + + This method processes scan results to build or update component entries. For each component: + + Component Counters (used by ComponentSummary): + - count: Incremented for each occurrence of the component + - declared: Incremented when component status is 'identified' + - undeclared: Incremented when component status is 'pending' + + License Tracking: + - For new components, initializes license dictionary through _append_component + - For existing components, updates license counters through _append_license_to_component + which tracks the number of occurrences of each license + + Args: + results: A dictionary containing the raw results of a component scan + Returns: + Updated components dictionary with file and snippet data + """ + for component in results.values(): + for c in component: + component_id = c.get('id') + if not component_id: + self.print_debug(f'WARNING: Result missing id. Skipping: {c}') + continue + ## Skip dependency + if component_id == ComponentID.DEPENDENCY.value: + continue + status = c.get('status') + if not status: + self.print_debug(f'WARNING: Result missing status. Skipping: {c}') + continue + if component_id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: + if not c.get('purl'): + self.print_debug(f'WARNING: Result missing purl. Skipping: {c}') + continue + if len(c.get('purl')) <= 0: + self.print_debug(f'WARNING: Result missing purls. Skipping: {c}') + continue + version = c.get('version') + if not version: + self.print_debug(f'WARNING: Result missing version. Setting it to unknown: {c}') + version = 'unknown' + c['version'] = version #If no version exists. Set 'unknown' version to current component + # Append component + components = self._append_component(components, c) + + # End component loop + # End components loop + return components + + def _get_dependencies_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract and process dependency components from results. + + :param results: A dictionary containing the raw results of a component scan + :param components: Existing components dictionary to update + :return: Updated components dictionary with dependency data + """ + for component in results.values(): + for c in component: + component_id = c.get('id') + if not component_id: + self.print_debug(f'WARNING: Result missing id. Skipping: {c}') + continue + status = c.get('status') + if not status: + self.print_debug(f'WARNING: Result missing status. Skipping: {c}') + continue + if component_id == ComponentID.DEPENDENCY.value: + if c.get('dependencies') is None: + continue + for dependency in c['dependencies']: + if not dependency.get('purl'): + self.print_debug(f'WARNING: Dependency result missing purl. Skipping: {dependency}') + continue + version = dependency.get('version') + if not version: + self.print_debug(f'WARNING: Result missing version. Setting it to unknown: {c}') + version = 'unknown' + c['version'] = version # Set an 'unknown' version to the current component + + # Append component + components = self._append_component(components, dependency) + + # End dependency loop + # End component loop + # End of result loop + return components + + def _load_input_file(self): + """ + Load the result.json file + + Returns: + Dict[str, Any]: The parsed JSON data + """ + if not os.path.exists(self.filepath): + self.print_stderr(f'ERROR: The file "{self.filepath}" does not exist.') + return None + with open(self.filepath, 'r') as jsonfile: + try: + return json.load(jsonfile) + except Exception as e: + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + return None + + def _convert_components_to_list(self, components: dict): + if components is None: + self.print_debug(f'WARNING: Components is empty {self.results}') + return None + results_list = list(components.values()) + for component in results_list: + licenses = component.get('licenses') + if licenses is not None: + component['licenses'] = list(licenses.values()) + else: + self.print_debug(f'WARNING: Licenses missing for: {component}') + component['licenses'] = [] + return results_list + + def _get_licenses_order_by_source_priority(self,licenses_data): + """ + Select licenses based on source priority: + 1. component_declared (highest priority) + 2. license_file + 3. file_header + 4. scancode (lowest priority) + + If any high-priority source is found, return only licenses from that source. + If none found, return all licenses. + + Returns: list with ordered licenses by source. + """ + # Define priority order (highest to lowest) + priority_sources = ['component_declared', 'license_file', 'file_header', 'scancode'] + + # Group licenses by source + licenses_by_source = {} + for license_item in licenses_data: + + source = license_item.get('source', 'unknown') + if source not in licenses_by_source: + licenses_by_source[source] = {} + + license_name = license_item.get('name') + if license_name: + # Use license name as key, store full license object as value + # If duplicate license names exist in same source, the last one wins + licenses_by_source[source][license_name] = license_item + + # Find the highest priority source that has licenses + for priority_source in priority_sources: + if priority_source in licenses_by_source: + self.print_trace(f'Choosing {priority_source} as source') + return list(licenses_by_source[priority_source].values()) + + # If no priority sources found, combine all licenses into a single list + self.print_debug("No priority sources found, returning all licenses as list") + return licenses_data + + +# +# End of PolicyCheck Class +# diff --git a/src/scanoss/inspection/license_summary.py b/src/scanoss/inspection/license_summary.py new file mode 100644 index 00000000..b3dc64a1 --- /dev/null +++ b/src/scanoss/inspection/license_summary.py @@ -0,0 +1,163 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +from typing import Any, Dict + +from .inspect_base import InspectBase + + +class LicenseSummary(InspectBase): + """ + SCANOSS LicenseSummary class + Inspects results and generates comprehensive license summaries from detected components. + + This class processes component scan results to extract, validate, and aggregate license + information, providing detailed summaries including copyleft analysis and license statistics. + """ + + # Define required license fields as class constants + REQUIRED_LICENSE_FIELDS = ['spdxid', 'url', 'copyleft', 'source'] + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = True, + quiet: bool = False, + filepath: str = None, + status: str = None, + output: str = None, + include: str = None, + exclude: str = None, + explicit: str = None, + ): + """ + Initialize the LicenseSummary class. + + :param debug: Enable debug mode + :param trace: Enable trace mode (default True) + :param quiet: Enable quiet mode + :param filepath: Path to the file containing component data + :param output: Path to save detailed output + :param include: Licenses to include in the analysis + :param exclude: Licenses to exclude from the analysis + :param explicit: Explicitly defined licenses + """ + super().__init__(debug, trace, quiet, filepath, output) + self.license_util.init(include, exclude, explicit) + self.filepath = filepath + self.output = output + self.status = status + self.include = include + self.exclude = exclude + self.explicit = explicit + + def _validate_license(self, license_data: Dict[str, Any]) -> bool: + """ + Validate that a license has all required fields. + + :param license_data: Dictionary containing license information + :return: True if license is valid, False otherwise + """ + for field in self.REQUIRED_LICENSE_FIELDS: + value = license_data.get(field) + if value is None: + self.print_debug(f'WARNING: {field} is empty in license: {license_data}') + return False + return True + + def _append_license(self, licenses: dict, new_license) -> None: + """Add or update a license in the licenses' dictionary.""" + spdxid = new_license.get("spdxid") + url = new_license.get("url") + copyleft = new_license.get("copyleft") + if spdxid not in licenses: + licenses[spdxid] = { + 'spdxid': spdxid, + 'url': url, + 'copyleft':copyleft, + 'count': new_license.get("count"), + } + else: + licenses[spdxid]['count'] += new_license.get("count") + + def _get_licenses_summary_from_components(self, components: list)-> dict: + """ + Get a license summary from detected components. + + :param components: List of all components + :return: Dict with license summary information + """ + licenses:dict = {} + licenses_with_copyleft = 0 + total_licenses = 0 + for component in components: + component_licenses = component.get("licenses", []) + for lic in component_licenses: + if not self._validate_license(lic): + continue + copyleft = lic.get("copyleft") + ## Increment counters + total_licenses += lic.get("count") + if copyleft: + licenses_with_copyleft += lic.get("count") + ## Add license + self._append_license(licenses, lic) + ## End for loop licenses + ## End for loop components + return { + 'licenses': list(licenses.values()), + 'total': total_licenses, + 'copyleft': licenses_with_copyleft + } + + + def _get_components(self): + """ + Extract and process components from results and their dependencies. + + This method performs the following steps: + 1. Validates that `self.results` is loaded. Returns `None` if not. + 2. Extracts file, snippet, and dependency components into a dictionary. + 3. Converts components to a list and processes their licenses. + + :return: A list of processed components with license data, or `None` if `self.results` is not set. + """ + if self.results is None: + return None + + components: dict = {} + # Extract component and license data from file and dependency results. Both helpers mutate `components` + self._get_components_data(self.results, components) + self._get_dependencies_data(self.results, components) + return self._convert_components_to_list(components) + + def run(self): + components = self._get_components() + license_summary = self._get_licenses_summary_from_components(components) + self.print_to_file_or_stdout(json.dumps(license_summary, indent=2), self.output) + return license_summary +# +# End of LicenseSummary Class +# \ No newline at end of file diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index 6c5d057d..77276758 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -1,7 +1,7 @@ """ SPDX-License-Identifier: MIT - Copyright (c) 2024, SCANOSS + Copyright (c) 2025, SCANOSS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,13 +22,11 @@ THE SOFTWARE. """ -import json -import os.path from abc import abstractmethod from enum import Enum from typing import Any, Callable, Dict, List -from ..scanossbase import ScanossBase +from .inspect_base import InspectBase from .utils.license_utils import LicenseUtil @@ -45,34 +43,11 @@ class PolicyStatus(Enum): SUCCESS = 0 FAIL = 1 ERROR = 2 - - # # End of PolicyStatus Class # - -class ComponentID(Enum): - """ - Enumeration representing different types of software components. - - Attributes: - FILE (str): Represents a file component (value: "file"). - SNIPPET (str): Represents a code snippet component (value: "snippet"). - DEPENDENCY (str): Represents a dependency component (value: "dependency"). - """ - - FILE = 'file' - SNIPPET = 'snippet' - DEPENDENCY = 'dependency' - - -# -# End of ComponentID Class -# - - -class PolicyCheck(ScanossBase): +class PolicyCheck(InspectBase): """ A base class for implementing various software policy checks. @@ -83,27 +58,24 @@ class PolicyCheck(ScanossBase): VALID_FORMATS (set): A set of valid output formats ('md', 'json'). Inherits from: - ScanossBase: A base class providing common functionality for SCANOSS-related operations. + InspectBase: A base class providing common functionality for SCANOSS-related operations. """ - VALID_FORMATS = {'md', 'json', 'jira_md'} - - def __init__( # noqa: PLR0913 - self, - debug: bool = False, - trace: bool = True, - quiet: bool = False, - filepath: str = None, - format_type: str = None, - status: str = None, - output: str = None, - name: str = None, + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = True, + quiet: bool = False, + filepath: str = None, + format_type: str = None, + status: str = None, + output: str = None, + name: str = None, ): - super().__init__(debug, trace, quiet) + super().__init__(debug, trace, quiet, filepath, output) self.license_util = LicenseUtil() self.filepath = filepath self.name = name - self.output = output self.format_type = format_type self.status = status self.results = self._load_input_file() @@ -166,162 +138,6 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: """ pass - @abstractmethod - def _get_components(self): - """ - Retrieve and process components from the preloaded results. - - This method performs the following steps: - 1. Checks if the results have been previously loaded (self.results). - 2. Extracts and processes components from the loaded results. - - :return: A list of processed components, or None if an error occurred during any step. - - Possible reasons for returning None include: - - Results not loaded (self.results is None) - - Failure to extract components from the results - - Note: - - This method assumes that the results have been previously loaded and stored in self.results. - - Implementations must extract components (e.g. via `_get_components_data`, - `_get_dependencies_data`, or other helpers). - - If `self.results` is `None`, simply return `None`. - """ - pass - - - def _append_component( - self, components: Dict[str, Any], new_component: Dict[str, Any], id: str, status: str - ) -> Dict[str, Any]: - """ - Append a new component to the component's dictionary. - - This function creates a new entry in the components dictionary for the given component, - or updates an existing entry if the component already exists. It also processes the - licenses associated with the component. - - :param components: The existing dictionary of components - :param new_component: The new component to be added or updated - :param id: The new component ID - :param status: The new component status - :return: The updated components dictionary - """ - # Determine the component key and purl based on component type - if id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: - purl = new_component['purl'][0] # Take the first purl for these component types - else: - purl = new_component['purl'] - - if not purl: - self.print_debug(f'WARNING: _append_component: No purl found for new component: {new_component}') - return components - - component_key = f'{purl}@{new_component["version"]}' - components[component_key] = { - 'purl': purl, - 'version': new_component['version'], - 'licenses': {}, - 'status': status, - } - if not new_component.get('licenses'): - self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}') - return components - - - licenses_order_by_source_priority = self._get_licenses_order_by_source_priority(new_component['licenses']) - # Process licenses for this component - for license_item in licenses_order_by_source_priority: - if license_item.get('name'): - spdxid = license_item['name'] - source = license_item.get('source') - if not source: - source = 'unknown' - components[component_key]['licenses'][spdxid] = { - 'spdxid': spdxid, - 'copyleft': self.license_util.is_copyleft(spdxid), - 'url': self.license_util.get_spdx_url(spdxid), - 'source': source, - } - return components - - def _get_components_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: - """ - Extract and process file and snippet components from results. - - :param results: A dictionary containing the raw results of a component scan - :param components: Existing components dictionary to update - :return: Updated components dictionary with file and snippet data - """ - for component in results.values(): - for c in component: - component_id = c.get('id') - if not component_id: - self.print_debug(f'WARNING: Result missing id. Skipping: {c}') - continue - ## Skip dependency - if component_id == ComponentID.DEPENDENCY.value: - continue - status = c.get('status') - if not status: - self.print_debug(f'WARNING: Result missing status. Skipping: {c}') - continue - if component_id in [ComponentID.FILE.value, ComponentID.SNIPPET.value]: - if not c.get('purl'): - self.print_debug(f'WARNING: Result missing purl. Skipping: {c}') - continue - if len(c.get('purl')) <= 0: - self.print_debug(f'WARNING: Result missing purls. Skipping: {c}') - continue - version = c.get('version') - if not version: - self.print_debug(f'WARNING: Result missing version. Setting it to unknown: {c}') - version = 'unknown' - c['version'] = version #If no version exists. Set 'unknown' version to current component - component_key = f'{c["purl"][0]}@{version}' - if component_key not in components: - components = self._append_component(components, c, component_id, status) - # End component loop - # End components loop - return components - - def _get_dependencies_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: - """ - Extract and process dependency components from results. - - :param results: A dictionary containing the raw results of a component scan - :param components: Existing components dictionary to update - :return: Updated components dictionary with dependency data - """ - for component in results.values(): - for c in component: - component_id = c.get('id') - if not component_id: - self.print_debug(f'WARNING: Result missing id. Skipping: {c}') - continue - status = c.get('status') - if not status: - self.print_debug(f'WARNING: Result missing status. Skipping: {c}') - continue - if component_id == ComponentID.DEPENDENCY.value: - if c.get('dependencies') is None: - continue - for dependency in c['dependencies']: - if not dependency.get('purl'): - self.print_debug(f'WARNING: Dependency result missing purl. Skipping: {dependency}') - continue - version = c.get('version') - if not version: - self.print_debug(f'WARNING: Result missing version. Setting it to unknown: {c}') - version = 'unknown' - c['version'] = version # If no version exists. Set 'unknown' version to current component - component_key = f'{dependency["purl"]}@{version}' - if component_key not in components: - components = self._append_component(components, dependency, component_id, status) - # End dependency loop - # End component loop - # End of result loop - return components - def generate_table(self, headers, rows, centered_columns=None): """ Generate a Markdown table. @@ -408,79 +224,6 @@ def _is_valid_format(self) -> bool: self.print_stderr(f'ERROR: Invalid format "{self.format_type}". Valid formats are: {valid_formats_str}') return False return True - - def _load_input_file(self): - """ - Load the result.json file - - Returns: - Dict[str, Any]: The parsed JSON data - """ - if not os.path.exists(self.filepath): - self.print_stderr(f'ERROR: The file "{self.filepath}" does not exist.') - return None - with open(self.filepath, 'r') as jsonfile: - try: - return json.load(jsonfile) - except Exception as e: - self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') - return None - - def _convert_components_to_list(self, components: dict): - if components is None: - self.print_debug(f'WARNING: Components is empty {self.results}') - return None - results_list = list(components.values()) - for component in results_list: - licenses = component.get('licenses') - if licenses is not None: - component['licenses'] = list(licenses.values()) - else: - self.print_debug(f'WARNING: Licenses missing for: {component}') - component['licenses'] = [] - return results_list - - def _get_licenses_order_by_source_priority(self,licenses_data): - """ - Select licenses based on source priority: - 1. component_declared (highest priority) - 2. license_file - 3. file_header - 4. scancode (lowest priority) - - If any high-priority source is found, return only licenses from that source. - If none found, return all licenses. - - Returns: list with ordered licenses by source. - """ - # Define priority order (highest to lowest) - priority_sources = ['component_declared', 'license_file', 'file_header', 'scancode'] - - # Group licenses by source - licenses_by_source = {} - for license_item in licenses_data: - - source = license_item.get('source', 'unknown') - if source not in licenses_by_source: - licenses_by_source[source] = {} - - license_name = license_item.get('name') - if license_name: - # Use license name as key, store full license object as value - # If duplicate license names exist in same source, the last one wins - licenses_by_source[source][license_name] = license_item - - # Find the highest priority source that has licenses - for priority_source in priority_sources: - if priority_source in licenses_by_source: - self.print_trace(f'Choosing {priority_source} as source') - return list(licenses_by_source[priority_source].values()) - - # If no priority sources found, combine all licenses into a single list - self.print_debug("No priority sources found, returning all licenses as list") - return licenses_data - - # # End of PolicyCheck Class -# +# \ No newline at end of file diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index f0c174fd..c53a4451 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -1,7 +1,7 @@ """ SPDX-License-Identifier: MIT - Copyright (c) 2024, SCANOSS + Copyright (c) 2025, SCANOSS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -79,7 +79,14 @@ def _get_undeclared_component(self, components: list) -> list or None: undeclared_components = [] for component in components: if component['status'] == 'pending': + # Remove unused keys del component['status'] + del component['count'] + del component['declared'] + del component['undeclared'] + for lic in component['licenses']: + lic.pop('count', None) # None is default value if key doesn't exist + lic.pop('source', None) # None is default value if key doesn't exist undeclared_components.append(component) # end component loop return undeclared_components @@ -148,12 +155,14 @@ def _json(self, components: list) -> Dict[str, Any]: :param components: List of undeclared components :return: Dictionary with formatted JSON details and summary """ + # Use component grouped by licenses to generate the summary + component_licenses = self._group_components_by_license(components) details = {} if len(components) > 0: details = {'components': components} return { 'details': f'{json.dumps(details, indent=2)}\n', - 'summary': self._get_summary(components), + 'summary': self._get_summary(component_licenses), } def _markdown(self, components: list) -> Dict[str, Any]: @@ -163,15 +172,15 @@ def _markdown(self, components: list) -> Dict[str, Any]: :param components: List of undeclared components :return: Dictionary with formatted Markdown details and summary """ - headers = ['Component', 'Version', 'License'] + headers = ['Component', 'License'] rows: [[]] = [] # TODO look at using SpdxLite license name lookup method - for component in components: - licenses = ' - '.join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) - rows.append([component['purl'], component['version'], licenses]) + component_licenses = self._group_components_by_license(components) + for component in component_licenses: + rows.append([component.get('purl'), component.get('license')]) return { 'details': f'### Undeclared components\n{self.generate_table(headers, rows)}\n', - 'summary': self._get_summary(components), + 'summary': self._get_summary(component_licenses), } def _jira_markdown(self, components: list) -> Dict[str, Any]: @@ -181,15 +190,15 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: :param components: List of undeclared components :return: Dictionary with formatted Markdown details and summary """ - headers = ['Component', 'Version', 'License'] + headers = ['Component', 'License'] rows: [[]] = [] # TODO look at using SpdxLite license name lookup method - for component in components: - licenses = ' - '.join(lic.get('spdxid', 'Unknown') for lic in component['licenses']) - rows.append([component['purl'], component['version'], licenses]) + component_licenses = self._group_components_by_license(components) + for component in component_licenses: + rows.append([component.get('purl'), component.get('license')]) return { 'details': f'{self.generate_jira_table(headers, rows)}', - 'summary': self._get_jira_summary(components), + 'summary': self._get_jira_summary(component_licenses), } def _get_unique_components(self, components: list) -> list: @@ -256,6 +265,36 @@ def _get_components(self): # Convert to list and process licenses return self._convert_components_to_list(components) + def _group_components_by_license(self,components): + """ + Groups components by their unique component-license pairs. + + This method processes a list of components and creates unique entries for each + component-license combination. If a component has multiple licenses, it will create + separate entries for each license. + + Args: + components: A list of component dictionaries. Each component should have: + - purl: Package URL identifying the component + - licenses: List of license dictionaries, each containing: + - spdxid: SPDX identifier for the license (optional) + + Returns: + list: A list of dictionaries, each containing: + - purl: The component's package URL + - license: The SPDX identifier of the license (or 'Unknown' if not provided) + """ + component_licenses: dict = {} + for component in components: + for lic in component['licenses']: + spdxid = lic.get('spdxid', 'Unknown') + key = f'{component["purl"]}-{spdxid}' + component_licenses[key] = { + 'purl': component['purl'], + 'license': spdxid, + } + return list(component_licenses.values()) + def run(self): """ Run the undeclared component inspection process. diff --git a/tests/data/empty-result.json b/tests/data/empty-result.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/tests/data/empty-result.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 771a7aa7..2bce4273 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -28,8 +28,11 @@ import unittest from scanoss.inspection.copyleft import Copyleft +from scanoss.inspection.license_summary import LicenseSummary from scanoss.inspection.undeclared_component import UndeclaredComponent +from src.scanoss.inspection.component_summary import ComponentSummary + class MyTestCase(unittest.TestCase): """ @@ -178,7 +181,7 @@ def test_undeclared_policy(self): status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] - expected_summary_output = """3 undeclared component(s) were found. + expected_summary_output = """2 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json { @@ -211,13 +214,12 @@ def test_undeclared_policy_markdown(self): details = results['details'] summary = results['summary'] expected_details_output = """ ### Undeclared components - | Component | Version | License | - | - | - | - | - | pkg:github/scanoss/scanner.c | 1.3.3 | GPL-2.0-only | - | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | - | pkg:github/scanoss/wfp | 6afc1f6 | GPL-2.0-only | """ + | Component | License | + | - | - | + | pkg:github/scanoss/scanner.c | GPL-2.0-only | + | pkg:github/scanoss/wfp | GPL-2.0-only | """ - expected_summary_output = """3 undeclared component(s) were found. + expected_summary_output = """2 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json { @@ -231,8 +233,6 @@ def test_undeclared_policy_markdown(self): ] }``` """ - - print(summary) self.assertEqual(status, 0) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) @@ -254,13 +254,12 @@ def test_undeclared_policy_markdown_scanoss_summary(self): details = results['details'] summary = results['summary'] expected_details_output = """ ### Undeclared components - | Component | Version | License | - | - | - | - | - | pkg:github/scanoss/scanner.c | 1.3.3 | GPL-2.0-only | - | pkg:github/scanoss/scanner.c | 1.1.4 | GPL-2.0-only | - | pkg:github/scanoss/wfp | 6afc1f6 | GPL-2.0-only | """ + | Component | License | + | - | - | + | pkg:github/scanoss/scanner.c | GPL-2.0-only | + | pkg:github/scanoss/wfp | GPL-2.0-only | """ - expected_summary_output = """3 undeclared component(s) were found. + expected_summary_output = """2 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file ```json @@ -277,8 +276,6 @@ def test_undeclared_policy_markdown_scanoss_summary(self): } } ```""" - - print(summary) self.assertEqual(status, 0) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) @@ -299,7 +296,7 @@ def test_undeclared_policy_scanoss_summary(self): status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] - expected_summary_output = """3 undeclared component(s) were found. + expected_summary_output = """2 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file ```json @@ -330,12 +327,11 @@ def test_undeclared_policy_jira_markdown_output(self): status, results = undeclared.run() details = results['details'] summary = results['summary'] - expected_details_output = """|*Component*|*Version*|*License*| -|pkg:github/scanoss/scanner.c|1.3.3|GPL-2.0-only| -|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only| -|pkg:github/scanoss/wfp|6afc1f6|GPL-2.0-only| + expected_details_output = """|*Component*|*License*| +|pkg:github/scanoss/scanner.c|GPL-2.0-only| +|pkg:github/scanoss/wfp|GPL-2.0-only| """ - expected_summary_output = """3 undeclared component(s) were found. + expected_summary_output = """2 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file {code:json} { @@ -373,6 +369,47 @@ def test_copyleft_policy_jira_markdown_output(self): self.assertEqual(status, 0) self.assertEqual(expected_details_output, details) + def test_inspect_license_summary(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + i_license_summary = LicenseSummary(filepath=input_file_name) + license_summary = i_license_summary.run() + self.assertEqual(license_summary['total'], 9) + self.assertEqual(license_summary['copyleft'], 7) + self.assertEqual(len(license_summary['licenses']), 2) + + def test_inspect_license_summary_with_empty_result(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'empty-result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + i_license_summary = LicenseSummary(filepath=input_file_name) + license_summary = i_license_summary.run() + self.assertEqual(license_summary['total'], 0) + self.assertEqual(license_summary['copyleft'], 0) + self.assertEqual(len(license_summary['licenses']), 0) + + def test_inspect_component_summary(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + i_component_summary = ComponentSummary(filepath=input_file_name) + component_summary = i_component_summary.run() + print(component_summary) + self.assertEqual(component_summary['total'], 7) + self.assertEqual(component_summary['undeclared'], 5) + self.assertEqual(component_summary['declared'], 2) + + def test_inspect_component_summary_empty_result(self): + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'empty-result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + i_component_summary = ComponentSummary(filepath=input_file_name) + component_summary = i_component_summary.run() + self.assertEqual(component_summary['total'], 0) + self.assertEqual(component_summary['undeclared'], 0) + self.assertEqual(component_summary['declared'], 0) + self.assertEqual(len(component_summary['components']), 0) if __name__ == '__main__': unittest.main() From 116a3b1e4c8e13b5b9fd4c057ffe8555d6ad7e0d Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Mon, 23 Jun 2025 09:34:04 -0300 Subject: [PATCH 344/489] Refactor inspect subcommand summary * chore:SP-2777 Refactor on inspect license and component summary * Upgrades version to v1.26.1 * chore:Updates CHANGELOG.md file --- CHANGELOG.md | 10 ++- src/scanoss/__init__.py | 2 +- src/scanoss/inspection/component_summary.py | 25 ++++--- src/scanoss/inspection/copyleft.py | 41 +++++------ src/scanoss/inspection/inspect_base.py | 34 +++++++++ src/scanoss/inspection/license_summary.py | 72 +++++++------------ .../inspection/undeclared_component.py | 35 +-------- tests/test_policy_inspect.py | 45 ++++++------ 8 files changed, 133 insertions(+), 131 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba5f9eca..1bb01570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.26.1] - 2025-06-23 + +### Added +- Added component count to inspect license summary +### Changed +- Modified summaries for inspect subcommand + ## [1.26.0] - 2025-06-20 ### Added - New `inspect license-summary` subcommand to generate license summaries from scan results @@ -552,4 +559,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.25.0]: https://github.com/scanoss/scanoss.py/compare/v1.24.0...v1.25.0 [1.25.1]: https://github.com/scanoss/scanoss.py/compare/v1.25.0...v1.25.1 [1.25.2]: https://github.com/scanoss/scanoss.py/compare/v1.25.1...v1.25.2 -[1.26.0]: https://github.com/scanoss/scanoss.py/compare/v1.25.2...v1.26.0 \ No newline at end of file +[1.26.0]: https://github.com/scanoss/scanoss.py/compare/v1.25.2...v1.26.0 +[1.26.1]: https://github.com/scanoss/scanoss.py/compare/v1.26.0...v1.26.1 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index a252dab9..d63417e3 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.26.0' +__version__ = '1.26.1' diff --git a/src/scanoss/inspection/component_summary.py b/src/scanoss/inspection/component_summary.py index ad25df2c..dc3d84db 100644 --- a/src/scanoss/inspection/component_summary.py +++ b/src/scanoss/inspection/component_summary.py @@ -35,12 +35,18 @@ def _get_component_summary_from_components(self, scan_components: list)-> dict: :param components: List of all components :return: Dict with license summary information """ + # A component is considered unique by its combination of PURL (Package URL) and license + component_licenses = self._group_components_by_license(scan_components) + total_components = len(component_licenses) + # Get undeclared components + undeclared_components = len([c for c in component_licenses if c['status'] == 'pending']) + components: list = [] - undeclared_components = 0 - total_components = 0 + total_undeclared_files = 0 + total_files_detected = 0 for component in scan_components: - total_components += component['count'] - undeclared_components += component['undeclared'] + total_files_detected += component['count'] + total_undeclared_files += component['undeclared'] components.append({ 'purl': component['purl'], 'version': component['version'], @@ -50,10 +56,13 @@ def _get_component_summary_from_components(self, scan_components: list)-> dict: }) ## End for loop components return { - 'components': components, - 'total': total_components, - 'undeclared': undeclared_components, - 'declared': total_components - undeclared_components, + "components": component_licenses, + 'totalComponents': total_components, + 'undeclaredComponents': undeclared_components, + 'declaredComponents': total_components - undeclared_components, + 'totalFilesDetected': total_files_detected, + 'totalFilesUndeclared': total_undeclared_files, + 'totalFilesDeclared': total_files_detected - total_undeclared_files, } def _get_components(self): diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index fbef3124..8caa9c98 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -78,12 +78,14 @@ def _json(self, components: list) -> Dict[str, Any]: :param components: List of components with copyleft licenses :return: Dictionary with formatted JSON details and summary """ + # A component is considered unique by its combination of PURL (Package URL) and license + component_licenses = self._group_components_by_license(components) details = {} if len(components) > 0: details = {'components': components} return { 'details': f'{json.dumps(details, indent=2)}\n', - 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n', + 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', } def _markdown(self, components: list) -> Dict[str, Any]: @@ -93,24 +95,24 @@ def _markdown(self, components: list) -> Dict[str, Any]: :param components: List of components with copyleft licenses :return: Dictionary with formatted Markdown details and summary """ - headers = ['Component', 'Version', 'License', 'URL', 'Copyleft'] + # A component is considered unique by its combination of PURL (Package URL) and license + component_licenses = self._group_components_by_license(components) + headers = ['Component', 'License', 'URL', 'Copyleft'] centered_columns = [1, 4] rows: [[]] = [] - for component in components: - for lic in component['licenses']: + for comp_lic_item in component_licenses: row = [ - component['purl'], - component['version'], - lic['spdxid'], - lic['url'], - 'YES' if lic['copyleft'] else 'NO', + comp_lic_item['purl'], + comp_lic_item['spdxid'], + comp_lic_item['url'], + 'YES' if comp_lic_item['copyleft'] else 'NO', ] rows.append(row) # End license loop # End component loop return { 'details': f'### Copyleft licenses\n{self.generate_table(headers, rows, centered_columns)}\n', - 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n', + 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', } def _jira_markdown(self, components: list) -> Dict[str, Any]: @@ -120,24 +122,24 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: :param components: List of components with copyleft licenses :return: Dictionary with formatted Markdown details and summary """ - headers = ['Component', 'Version', 'License', 'URL', 'Copyleft'] + # A component is considered unique by its combination of PURL (Package URL) and license + component_licenses = self._group_components_by_license(components) + headers = ['Component', 'License', 'URL', 'Copyleft'] centered_columns = [1, 4] rows: [[]] = [] - for component in components: - for lic in component['licenses']: + for comp_lic_item in component_licenses: row = [ - component['purl'], - component['version'], - lic['spdxid'], - lic['url'], - 'YES' if lic['copyleft'] else 'NO', + comp_lic_item['purl'], + comp_lic_item['spdxid'], + comp_lic_item['url'], + 'YES' if comp_lic_item['copyleft'] else 'NO', ] rows.append(row) # End license loop # End component loop return { 'details': f'{self.generate_jira_table(headers, rows, centered_columns)}', - 'summary': f'{len(components)} component(s) with copyleft licenses were found.\n', + 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', } def _filter_components_with_copyleft_licenses(self, components: list) -> list: @@ -161,7 +163,6 @@ def _filter_components_with_copyleft_licenses(self, components: list) -> list: lic.pop('count', None) # None is default value if key doesn't exist filtered_component['licenses'] = copyleft_licenses - del filtered_component['status'] filtered_components.append(filtered_component) # End component loop self.print_debug(f'Copyleft components: {filtered_components}') diff --git a/src/scanoss/inspection/inspect_base.py b/src/scanoss/inspection/inspect_base.py index df8c2fa4..927203ff 100644 --- a/src/scanoss/inspection/inspect_base.py +++ b/src/scanoss/inspection/inspect_base.py @@ -372,6 +372,40 @@ def _get_licenses_order_by_source_priority(self,licenses_data): self.print_debug("No priority sources found, returning all licenses as list") return licenses_data + def _group_components_by_license(self,components): + """ + Groups components by their unique component-license pairs. + + This method processes a list of components and creates unique entries for each + component-license combination. If a component has multiple licenses, it will create + separate entries for each license. + + Args: + components: A list of component dictionaries. Each component should have: + - purl: Package URL identifying the component + - licenses: List of license dictionaries, each containing: + - spdxid: SPDX identifier for the license (optional) + + Returns: + list: A list of dictionaries, each containing: + - purl: The component's package URL + - license: The SPDX identifier of the license (or 'Unknown' if not provided) + """ + component_licenses: dict = {} + for component in components: + for lic in component['licenses']: + spdxid = lic.get('spdxid', 'Unknown') + if spdxid not in component_licenses: + key = f'{component["purl"]}-{spdxid}' + component_licenses[key] = { + 'purl': component['purl'], + 'spdxid': spdxid, + 'status': component['status'], + 'copyleft': lic['copyleft'], + 'url': lic['url'], + } + return list(component_licenses.values()) + # # End of PolicyCheck Class diff --git a/src/scanoss/inspection/license_summary.py b/src/scanoss/inspection/license_summary.py index b3dc64a1..6b15b86a 100644 --- a/src/scanoss/inspection/license_summary.py +++ b/src/scanoss/inspection/license_summary.py @@ -23,7 +23,6 @@ """ import json -from typing import Any, Dict from .inspect_base import InspectBase @@ -73,35 +72,6 @@ def __init__( # noqa: PLR0913 self.exclude = exclude self.explicit = explicit - def _validate_license(self, license_data: Dict[str, Any]) -> bool: - """ - Validate that a license has all required fields. - - :param license_data: Dictionary containing license information - :return: True if license is valid, False otherwise - """ - for field in self.REQUIRED_LICENSE_FIELDS: - value = license_data.get(field) - if value is None: - self.print_debug(f'WARNING: {field} is empty in license: {license_data}') - return False - return True - - def _append_license(self, licenses: dict, new_license) -> None: - """Add or update a license in the licenses' dictionary.""" - spdxid = new_license.get("spdxid") - url = new_license.get("url") - copyleft = new_license.get("copyleft") - if spdxid not in licenses: - licenses[spdxid] = { - 'spdxid': spdxid, - 'url': url, - 'copyleft':copyleft, - 'count': new_license.get("count"), - } - else: - licenses[spdxid]['count'] += new_license.get("count") - def _get_licenses_summary_from_components(self, components: list)-> dict: """ Get a license summary from detected components. @@ -109,27 +79,35 @@ def _get_licenses_summary_from_components(self, components: list)-> dict: :param components: List of all components :return: Dict with license summary information """ + # A component is considered unique by its combination of PURL (Package URL) and license + component_licenses = self._group_components_by_license(components) + license_component_count = {} + # Count license per component + for lic in component_licenses: + if lic['spdxid'] not in license_component_count: + license_component_count[lic['spdxid']] = 1 + else: + license_component_count[lic['spdxid']] += 1 licenses:dict = {} - licenses_with_copyleft = 0 - total_licenses = 0 - for component in components: - component_licenses = component.get("licenses", []) - for lic in component_licenses: - if not self._validate_license(lic): - continue - copyleft = lic.get("copyleft") - ## Increment counters - total_licenses += lic.get("count") - if copyleft: - licenses_with_copyleft += lic.get("count") - ## Add license - self._append_license(licenses, lic) + for comp_lic in component_licenses: + spdxid = comp_lic.get("spdxid") + url = comp_lic.get("url") + copyleft = comp_lic.get("copyleft") + if spdxid not in licenses: + licenses[spdxid] = { + 'spdxid': spdxid, + 'url': url, + 'copyleft': copyleft, + 'componentCount': license_component_count.get(spdxid, 0), # Append component count to license + } ## End for loop licenses ## End for loop components + detected_licenses = list(licenses.values()) + licenses_with_copyleft = [lic for lic in detected_licenses if lic['copyleft']] return { - 'licenses': list(licenses.values()), - 'total': total_licenses, - 'copyleft': licenses_with_copyleft + 'licenses': detected_licenses, + 'detectedLicenses': len(detected_licenses), # Count unique licenses. SPDXID is considered unique + 'detectedLicensesWithCopyleft': len(licenses_with_copyleft), } diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index c53a4451..121a367a 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -80,7 +80,6 @@ def _get_undeclared_component(self, components: list) -> list or None: for component in components: if component['status'] == 'pending': # Remove unused keys - del component['status'] del component['count'] del component['declared'] del component['undeclared'] @@ -177,7 +176,7 @@ def _markdown(self, components: list) -> Dict[str, Any]: # TODO look at using SpdxLite license name lookup method component_licenses = self._group_components_by_license(components) for component in component_licenses: - rows.append([component.get('purl'), component.get('license')]) + rows.append([component.get('purl'), component.get('spdxid')]) return { 'details': f'### Undeclared components\n{self.generate_table(headers, rows)}\n', 'summary': self._get_summary(component_licenses), @@ -195,7 +194,7 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: # TODO look at using SpdxLite license name lookup method component_licenses = self._group_components_by_license(components) for component in component_licenses: - rows.append([component.get('purl'), component.get('license')]) + rows.append([component.get('purl'), component.get('spdxid')]) return { 'details': f'{self.generate_jira_table(headers, rows)}', 'summary': self._get_jira_summary(component_licenses), @@ -265,36 +264,6 @@ def _get_components(self): # Convert to list and process licenses return self._convert_components_to_list(components) - def _group_components_by_license(self,components): - """ - Groups components by their unique component-license pairs. - - This method processes a list of components and creates unique entries for each - component-license combination. If a component has multiple licenses, it will create - separate entries for each license. - - Args: - components: A list of component dictionaries. Each component should have: - - purl: Package URL identifying the component - - licenses: List of license dictionaries, each containing: - - spdxid: SPDX identifier for the license (optional) - - Returns: - list: A list of dictionaries, each containing: - - purl: The component's package URL - - license: The SPDX identifier of the license (or 'Unknown' if not provided) - """ - component_licenses: dict = {} - for component in components: - for lic in component['licenses']: - spdxid = lic.get('spdxid', 'Unknown') - key = f'{component["purl"]}-{spdxid}' - component_licenses[key] = { - 'purl': component['purl'], - 'license': spdxid, - } - return list(component_licenses.values()) - def run(self): """ Run the undeclared component inspection process. diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 2bce4273..797fff1b 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -145,10 +145,10 @@ def test_copyleft_policy_markdown(self): copyleft = Copyleft(filepath=input_file_name, format_type='md', explicit='MIT') status, results = copyleft.run() expected_detail_output = ( - '### Copyleft licenses \n | Component | Version | License | URL | Copyleft |\n' - ' | - | :-: | - | - | :-: |\n' - ' | pkg:npm/%40electron/rebuild | 3.7.0 | MIT | https://spdx.org/licenses/MIT.html | YES |\n' - '| pkg:npm/%40emotion/react | 11.13.3 | MIT | https://spdx.org/licenses/MIT.html | YES | \n' + '### Copyleft licenses \n | Component | License | URL | Copyleft |\n' + ' | - | :-: | - | - |\n' + ' | pkg:npm/%40electron/rebuild | MIT | https://spdx.org/licenses/MIT.html | YES |\n' + '| pkg:npm/%40emotion/react | MIT | https://spdx.org/licenses/MIT.html | YES | \n' ) expected_summary_output = '2 component(s) with copyleft licenses were found.\n' self.assertEqual( @@ -359,12 +359,10 @@ def test_copyleft_policy_jira_markdown_output(self): copyleft = Copyleft(filepath=input_file_name, format_type='jira_md') status, results = copyleft.run() details = results['details'] - expected_details_output = """|*Component*|*Version*|*License*|*URL*|*Copyleft*| -|pkg:github/scanoss/scanner.c|1.3.3|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| -|pkg:github/scanoss/scanner.c|1.1.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| -|pkg:github/scanoss/engine|5.4.0|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| -|pkg:github/scanoss/wfp|6afc1f6|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| -|pkg:github/scanoss/engine|4.0.4|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| + expected_details_output = """|*Component*|*License*|*URL*|*Copyleft*| +|pkg:github/scanoss/scanner.c|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| +|pkg:github/scanoss/engine|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| +|pkg:github/scanoss/wfp|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| """ self.assertEqual(status, 0) self.assertEqual(expected_details_output, details) @@ -375,9 +373,8 @@ def test_inspect_license_summary(self): input_file_name = os.path.join(script_dir, 'data', file_name) i_license_summary = LicenseSummary(filepath=input_file_name) license_summary = i_license_summary.run() - self.assertEqual(license_summary['total'], 9) - self.assertEqual(license_summary['copyleft'], 7) - self.assertEqual(len(license_summary['licenses']), 2) + self.assertEqual(license_summary['detectedLicenses'], 2) + self.assertEqual(license_summary['detectedLicensesWithCopyleft'], 1) def test_inspect_license_summary_with_empty_result(self): script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -385,8 +382,8 @@ def test_inspect_license_summary_with_empty_result(self): input_file_name = os.path.join(script_dir, 'data', file_name) i_license_summary = LicenseSummary(filepath=input_file_name) license_summary = i_license_summary.run() - self.assertEqual(license_summary['total'], 0) - self.assertEqual(license_summary['copyleft'], 0) + self.assertEqual(license_summary['detectedLicenses'], 0) + self.assertEqual(license_summary['detectedLicensesWithCopyleft'], 0) self.assertEqual(len(license_summary['licenses']), 0) def test_inspect_component_summary(self): @@ -396,9 +393,12 @@ def test_inspect_component_summary(self): i_component_summary = ComponentSummary(filepath=input_file_name) component_summary = i_component_summary.run() print(component_summary) - self.assertEqual(component_summary['total'], 7) - self.assertEqual(component_summary['undeclared'], 5) - self.assertEqual(component_summary['declared'], 2) + self.assertEqual(component_summary['totalComponents'], 3) + self.assertEqual(component_summary['undeclaredComponents'], 2) + self.assertEqual(component_summary['declaredComponents'], 1) + self.assertEqual(component_summary['totalFilesDetected'], 7) + self.assertEqual(component_summary['totalFilesUndeclared'], 5) + self.assertEqual(component_summary['totalFilesDeclared'], 2) def test_inspect_component_summary_empty_result(self): script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -406,10 +406,13 @@ def test_inspect_component_summary_empty_result(self): input_file_name = os.path.join(script_dir, 'data', file_name) i_component_summary = ComponentSummary(filepath=input_file_name) component_summary = i_component_summary.run() - self.assertEqual(component_summary['total'], 0) - self.assertEqual(component_summary['undeclared'], 0) - self.assertEqual(component_summary['declared'], 0) + self.assertEqual(component_summary['totalComponents'], 0) + self.assertEqual(component_summary['undeclaredComponents'], 0) + self.assertEqual(component_summary['declaredComponents'], 0) self.assertEqual(len(component_summary['components']), 0) + self.assertEqual(component_summary['totalFilesDetected'], 0) + self.assertEqual(component_summary['totalFilesUndeclared'], 0) + self.assertEqual(component_summary['totalFilesDeclared'], 0) if __name__ == '__main__': unittest.main() From 663ef16c7673a3c9822f1b91bff613bc0505a36e Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Tue, 24 Jun 2025 15:12:03 -0300 Subject: [PATCH 345/489] bug: Fix inspection of undeclared components with empty licenses * bug:SP-2804 Fix inspection of undeclared components with empty licenses * chore:Upgrades version to v1.26.2 * chore:Updates CHANGELOG.md file --- CHANGELOG.md | 8 +++-- src/scanoss/__init__.py | 2 +- src/scanoss/inspection/inspect_base.py | 27 ++++++++++++++--- tests/data/result.json | 32 ++++++++++++++++++++ tests/test_policy_inspect.py | 42 ++++++++++++++++++-------- tests/test_spdxlite.py | 4 +-- 6 files changed, 93 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bb01570..0c368a73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.26.1] - 2025-06-23 +## [1.26.2] - 2025-06-24 +### Fixed +- Fixed inspection of undeclared components with empty licenses +## [1.26.1] - 2025-06-23 ### Added - Added component count to inspect license summary ### Changed @@ -560,4 +563,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.25.1]: https://github.com/scanoss/scanoss.py/compare/v1.25.0...v1.25.1 [1.25.2]: https://github.com/scanoss/scanoss.py/compare/v1.25.1...v1.25.2 [1.26.0]: https://github.com/scanoss/scanoss.py/compare/v1.25.2...v1.26.0 -[1.26.1]: https://github.com/scanoss/scanoss.py/compare/v1.26.0...v1.26.1 \ No newline at end of file +[1.26.1]: https://github.com/scanoss/scanoss.py/compare/v1.26.0...v1.26.1 +[1.26.2]: https://github.com/scanoss/scanoss.py/compare/v1.26.1...v1.26.2 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index d63417e3..2e6162d2 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.26.1' +__version__ = '1.26.2' diff --git a/src/scanoss/inspection/inspect_base.py b/src/scanoss/inspection/inspect_base.py index 927203ff..03923b6c 100644 --- a/src/scanoss/inspection/inspect_base.py +++ b/src/scanoss/inspection/inspect_base.py @@ -393,14 +393,31 @@ def _group_components_by_license(self,components): """ component_licenses: dict = {} for component in components: - for lic in component['licenses']: - spdxid = lic.get('spdxid', 'Unknown') + purl = component.get('purl', '') + status = component.get('status', '') + licenses = component.get('licenses', []) + + # Component without license + if not licenses: + key = f'{purl}-unknown' + component_licenses[key] = { + 'purl': purl, + 'spdxid': 'unknown', + 'status': status, + 'copyleft': False, + 'url': '-', + } + continue + + # Iterate over licenses component licenses + for lic in licenses: + spdxid = lic.get('spdxid', 'unknown') if spdxid not in component_licenses: - key = f'{component["purl"]}-{spdxid}' + key = f'{purl}-{spdxid}' component_licenses[key] = { - 'purl': component['purl'], + 'purl': purl, 'spdxid': spdxid, - 'status': component['status'], + 'status': status, 'copyleft': lic['copyleft'], 'url': lic['url'], } diff --git a/tests/data/result.json b/tests/data/result.json index fcf0df98..00916605 100644 --- a/tests/data/result.json +++ b/tests/data/result.json @@ -11,6 +11,38 @@ } } ], + "inc/log.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/inc/json.h", + "file_hash": "e91a03b850651dd56dd979ba92668a19", + "file_url": "https://api.osskb.org/file_contents/e91a03b850651dd56dd979ba92668a19", + "id": "file", + "latest": "1.3.4", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/jenkins-pipeline-example" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "e91a03b850651dd56dd979ba92668a19", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], "inc/json.h": [ { "component": "scanner.c", diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 797fff1b..f0e8f1ac 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -181,11 +181,14 @@ def test_undeclared_policy(self): status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] - expected_summary_output = """2 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json { "components":[ + { + "purl": "pkg:github/scanoss/jenkins-pipeline-example" + }, { "purl": "pkg:github/scanoss/scanner.c" }, @@ -195,7 +198,7 @@ def test_undeclared_policy(self): ] }``` """ - self.assertEqual(len(details['components']), 3) + self.assertEqual(len(details['components']), 4) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) ) @@ -216,14 +219,18 @@ def test_undeclared_policy_markdown(self): expected_details_output = """ ### Undeclared components | Component | License | | - | - | + | pkg:github/scanoss/jenkins-pipeline-example | unknown | | pkg:github/scanoss/scanner.c | GPL-2.0-only | | pkg:github/scanoss/wfp | GPL-2.0-only | """ - expected_summary_output = """2 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json { "components":[ + { + "purl": "pkg:github/scanoss/jenkins-pipeline-example" + }, { "purl": "pkg:github/scanoss/scanner.c" }, @@ -256,16 +263,20 @@ def test_undeclared_policy_markdown_scanoss_summary(self): expected_details_output = """ ### Undeclared components | Component | License | | - | - | + | pkg:github/scanoss/jenkins-pipeline-example | unknown | | pkg:github/scanoss/scanner.c | GPL-2.0-only | | pkg:github/scanoss/wfp | GPL-2.0-only | """ - expected_summary_output = """2 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file ```json { "bom": { "include": [ + { + "purl": "pkg:github/scanoss/jenkins-pipeline-example" + }, { "purl": "pkg:github/scanoss/scanner.c" }, @@ -296,13 +307,16 @@ def test_undeclared_policy_scanoss_summary(self): status, results = undeclared.run() details = json.loads(results['details']) summary = results['summary'] - expected_summary_output = """2 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file ```json { "bom": { "include": [ + { + "purl": "pkg:github/scanoss/jenkins-pipeline-example" + }, { "purl": "pkg:github/scanoss/scanner.c" }, @@ -314,7 +328,7 @@ def test_undeclared_policy_scanoss_summary(self): } ```""" self.assertEqual(status, 0) - self.assertEqual(len(details['components']), 3) + self.assertEqual(len(details['components']), 4) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) ) @@ -328,15 +342,19 @@ def test_undeclared_policy_jira_markdown_output(self): details = results['details'] summary = results['summary'] expected_details_output = """|*Component*|*License*| +|pkg:github/scanoss/jenkins-pipeline-example|unknown| |pkg:github/scanoss/scanner.c|GPL-2.0-only| |pkg:github/scanoss/wfp|GPL-2.0-only| """ - expected_summary_output = """2 undeclared component(s) were found. + expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file {code:json} { "bom": { "include": [ + { + "purl": "pkg:github/scanoss/jenkins-pipeline-example" + }, { "purl": "pkg:github/scanoss/scanner.c" }, @@ -373,7 +391,7 @@ def test_inspect_license_summary(self): input_file_name = os.path.join(script_dir, 'data', file_name) i_license_summary = LicenseSummary(filepath=input_file_name) license_summary = i_license_summary.run() - self.assertEqual(license_summary['detectedLicenses'], 2) + self.assertEqual(license_summary['detectedLicenses'], 3) self.assertEqual(license_summary['detectedLicensesWithCopyleft'], 1) def test_inspect_license_summary_with_empty_result(self): @@ -393,11 +411,11 @@ def test_inspect_component_summary(self): i_component_summary = ComponentSummary(filepath=input_file_name) component_summary = i_component_summary.run() print(component_summary) - self.assertEqual(component_summary['totalComponents'], 3) - self.assertEqual(component_summary['undeclaredComponents'], 2) + self.assertEqual(component_summary['totalComponents'], 4) + self.assertEqual(component_summary['undeclaredComponents'], 3) self.assertEqual(component_summary['declaredComponents'], 1) - self.assertEqual(component_summary['totalFilesDetected'], 7) - self.assertEqual(component_summary['totalFilesUndeclared'], 5) + self.assertEqual(component_summary['totalFilesDetected'], 8) + self.assertEqual(component_summary['totalFilesUndeclared'], 6) self.assertEqual(component_summary['totalFilesDeclared'], 2) def test_inspect_component_summary_empty_result(self): diff --git a/tests/test_spdxlite.py b/tests/test_spdxlite.py index 762dc58a..6c7ed3f3 100644 --- a/tests/test_spdxlite.py +++ b/tests/test_spdxlite.py @@ -58,8 +58,8 @@ def testSpdxLite(self): self.assertEqual(name, "SCANOSS-SBOM") self.assertEqual(organization, "Organization: SCANOSS") self.assertEqual(creation_info_comment, "SBOM Build information - SBOM Type: Build") - self.assertEqual(len(document_describes), 5) - self.assertEqual(len(packages), 5) + self.assertEqual(len(document_describes), 6) + self.assertEqual(len(packages), 6) for package in packages: for checksum in package.get("checksums", []): From 2dca59ac445e23bce5130dc26b558e0a3b3a2f8a Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Thu, 26 Jun 2025 12:45:39 -0300 Subject: [PATCH 346/489] bug:SP-2830 fixes bug on inspect subcommand on components with empty licenses --- src/scanoss/inspection/copyleft.py | 2 +- src/scanoss/inspection/inspect_base.py | 10 +-- src/scanoss/inspection/license_summary.py | 2 +- src/scanoss/inspection/policy_check.py | 2 +- .../inspection/undeclared_component.py | 2 +- tests/data/result.json | 64 +++++++++++++++++++ tests/test_policy_inspect.py | 4 +- 7 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/copyleft.py index 8caa9c98..9760e763 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/copyleft.py @@ -37,7 +37,7 @@ class Copyleft(PolicyCheck): def __init__( # noqa: PLR0913 self, debug: bool = False, - trace: bool = True, + trace: bool = False, quiet: bool = False, filepath: str = None, format_type: str = 'json', diff --git a/src/scanoss/inspection/inspect_base.py b/src/scanoss/inspection/inspect_base.py index 03923b6c..5384ee6c 100644 --- a/src/scanoss/inspection/inspect_base.py +++ b/src/scanoss/inspection/inspect_base.py @@ -66,7 +66,7 @@ class InspectBase(ScanossBase): def __init__( # noqa: PLR0913 self, debug: bool = False, - trace: bool = True, + trace: bool = False, quiet: bool = False, filepath: str = None, output: str = None, @@ -152,9 +152,6 @@ def _append_component(self, components: Dict[str, Any], new_component: Dict[str, 'declared': 1 if status == 'identified' else 0, 'undeclared': 1 if status == 'pending' else 0 } - if not new_component.get('licenses'): - self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}') - return components ## Append license to component self._append_license_to_component(components, new_component, component_key) @@ -179,6 +176,11 @@ def _append_license_to_component(self, new_component: Component whose licenses need to be processed component_key: purl + version of the component to be updated """ + # If not licenses are present + if not new_component.get('licenses'): + self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}') + return + licenses_order_by_source_priority = self._get_licenses_order_by_source_priority(new_component['licenses']) # Process licenses for this component for license_item in licenses_order_by_source_priority: diff --git a/src/scanoss/inspection/license_summary.py b/src/scanoss/inspection/license_summary.py index 6b15b86a..80e06a47 100644 --- a/src/scanoss/inspection/license_summary.py +++ b/src/scanoss/inspection/license_summary.py @@ -42,7 +42,7 @@ class LicenseSummary(InspectBase): def __init__( # noqa: PLR0913 self, debug: bool = False, - trace: bool = True, + trace: bool = False, quiet: bool = False, filepath: str = None, status: str = None, diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index 77276758..aba8185a 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -64,7 +64,7 @@ class PolicyCheck(InspectBase): def __init__( # noqa: PLR0913 self, debug: bool = False, - trace: bool = True, + trace: bool = False, quiet: bool = False, filepath: str = None, format_type: str = None, diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/undeclared_component.py index 121a367a..1f221f55 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/undeclared_component.py @@ -37,7 +37,7 @@ class UndeclaredComponent(PolicyCheck): def __init__( # noqa: PLR0913 self, debug: bool = False, - trace: bool = True, + trace: bool = False, quiet: bool = False, filepath: str = None, format_type: str = 'json', diff --git a/tests/data/result.json b/tests/data/result.json index 00916605..3df4c7dc 100644 --- a/tests/data/result.json +++ b/tests/data/result.json @@ -104,6 +104,70 @@ "version": "1.3.3" } ], + "inc/component.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/inc/json.h", + "file_hash": "e91a03b850651dd56dd979ba92668a19", + "file_url": "https://api.osskb.org/file_contents/e91a03b850651dd56dd979ba92668a19", + "id": "file", + "latest": "1.3.4", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "e91a03b850651dd56dd979ba92668a19", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], + "inc/helper.c": [ + { + "component": "scanner.c", + "file": "scanner.c-1.3.3/external/inc/json.h", + "file_hash": "e91a03b850651dd56dd979ba92668a19", + "file_url": "https://api.osskb.org/file_contents/e91a03b850651dd56dd979ba92668a19", + "id": "file", + "latest": "1.3.4", + "licenses": [], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/scanoss/scanner.c" + ], + "release_date": "2021-05-26", + "server": { + "kb_version": { + "daily": "24.08.16", + "monthly": "24.07" + }, + "version": "5.4.8" + }, + "source_hash": "e91a03b850651dd56dd979ba92668a19", + "status": "pending", + "url": "https://github.com/scanoss/scanner.c", + "url_hash": "2d1700ba496453d779d4987255feb5f2", + "url_stats": {}, + "vendor": "scanoss", + "version": "1.3.3" + } + ], "inc/log.h": [ { "component": "scanner.c", diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index f0e8f1ac..1f2cab5d 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -414,8 +414,8 @@ def test_inspect_component_summary(self): self.assertEqual(component_summary['totalComponents'], 4) self.assertEqual(component_summary['undeclaredComponents'], 3) self.assertEqual(component_summary['declaredComponents'], 1) - self.assertEqual(component_summary['totalFilesDetected'], 8) - self.assertEqual(component_summary['totalFilesUndeclared'], 6) + self.assertEqual(component_summary['totalFilesDetected'], 10) + self.assertEqual(component_summary['totalFilesUndeclared'], 8) self.assertEqual(component_summary['totalFilesDeclared'], 2) def test_inspect_component_summary_empty_result(self): From 2e1ce6f641b2c98df931642605baa35db0687a46 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Thu, 26 Jun 2025 12:46:31 -0300 Subject: [PATCH 347/489] chore: Upgrades app version to v1.26.3 --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 2e6162d2..55786eb0 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.26.2' +__version__ = '1.26.3' From b66d05412c8caab383e937814b8601bde30b2ca2 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Thu, 26 Jun 2025 12:50:42 -0300 Subject: [PATCH 348/489] chore:Updates CHANGELOG.md file --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c368a73..113cbca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.26.3] - 2025-06-26 +### Fixed +- Fixed crash in inspect subcommand when processing components that lack license information +- Set the default trace value to false for inspect command + ## [1.26.2] - 2025-06-24 ### Fixed - Fixed inspection of undeclared components with empty licenses @@ -564,4 +569,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.25.2]: https://github.com/scanoss/scanoss.py/compare/v1.25.1...v1.25.2 [1.26.0]: https://github.com/scanoss/scanoss.py/compare/v1.25.2...v1.26.0 [1.26.1]: https://github.com/scanoss/scanoss.py/compare/v1.26.0...v1.26.1 -[1.26.2]: https://github.com/scanoss/scanoss.py/compare/v1.26.1...v1.26.2 \ No newline at end of file +[1.26.2]: https://github.com/scanoss/scanoss.py/compare/v1.26.1...v1.26.2 +[1.26.3]: https://github.com/scanoss/scanoss.py/compare/v1.26.2...v1.26.3 \ No newline at end of file From 73db5ac0b0d72cb5a2f64885b0a5130a36bb297c Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 22 May 2025 11:10:56 +0200 Subject: [PATCH 349/489] [SP-2587] Add directory simhash, modify concatenated names to remove extensions --- src/scanoss/file_filters.py | 9 +++-- src/scanoss/scanners/folder_hasher.py | 52 +++++++++++++++++---------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index f7c1950b..d52ea386 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -25,7 +25,7 @@ import os import sys from pathlib import Path -from typing import List +from typing import List, Optional from pathspec import GitIgnoreSpec @@ -511,7 +511,7 @@ def get_filtered_files_from_folder(self, root: str) -> List[str]: # Now filter the files and return the reduced list return self.get_filtered_files_from_files(all_files, str(root_path)) - def get_filtered_files_from_files(self, files: List[str], scan_root: str = None) -> List[str]: + def get_filtered_files_from_files(self, files: List[str], scan_root: Optional[str] = None) -> List[str]: """ Retrieve a list of files to scan or fingerprint from a given list of files based on filter settings. @@ -615,8 +615,13 @@ def _get_operation_patterns(self, operation_type: str) -> List[str]: # Default patterns for skipping directories if not self.all_folders: DEFAULT_SKIPPED_DIR_LIST = DEFAULT_SKIPPED_DIRS_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_DIRS + DEFAULT_SKIPPED_DIR_EXT_LIST = ( + DEFAULT_SKIPPED_DIR_EXT_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_DIR_EXT + ) for dir_name in DEFAULT_SKIPPED_DIR_LIST: patterns.append(f'{dir_name}/') + for dir_extension in DEFAULT_SKIPPED_DIR_EXT_LIST: + patterns.append(f'*{dir_extension}/') # Custom patterns added in SCANOSS settings file if self.scanoss_settings: diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index f8654859..9b59e837 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -35,7 +35,7 @@ class DirectoryFile: Represents a file in the directory tree for folder hashing. """ - def __init__(self, path: str, key: bytes, key_str: str): + def __init__(self, path: str, key: List[bytes], key_str: str): self.path = path self.key = key self.key_str = key_str @@ -77,7 +77,7 @@ class FolderHasher: def __init__( self, scan_dir: str, - config: Optional[FolderHasherConfig] = None, + config: FolderHasherConfig, scanoss_settings: Optional[ScanossSettings] = None, ): self.base = ScanossBase( @@ -199,6 +199,7 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: 'path_id': node.path, 'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None, 'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None, + 'sim_hash_dir': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None, 'children': [self._hash_calc_from_node(child) for child in node.children.values()], } @@ -218,6 +219,8 @@ def _hash_calc(self, node: DirectoryNode) -> dict: dict: A dictionary with 'name_hash' and 'content_hash' keys. """ processed_hashes = set() + unique_file_names = set() + unique_directories = set() file_hashes = [] selected_names = [] @@ -225,37 +228,48 @@ def _hash_calc(self, node: DirectoryNode) -> dict: key_str = file.key_str if key_str in processed_hashes: continue - processed_hashes.add(key_str) - selected_names.append(os.path.basename(file.path)) + file_name = os.path.basename(file.path) + file_name_without_extension, _ = os.path.splitext(file_name) + current_directory = os.path.dirname(file.path) + + last_directory = os.path.basename(current_directory) - file_key = bytes(file.key) - file_hashes.append(file_key) + if last_directory == '': + last_directory = os.path.basename(os.getcwd()) + + processed_hashes.add(key_str) + unique_file_names.add(file_name_without_extension) + unique_directories.add(last_directory) + selected_names.append(file_name) + file_hashes.append(file.key) if len(selected_names) < MINIMUM_FILE_COUNT: - return { - 'name_hash': None, - 'content_hash': None, - } + return {'name_hash': None, 'content_hash': None, 'dir_hash': None} selected_names.sort() concatenated_names = ''.join(selected_names) if len(concatenated_names.encode('utf-8')) < MINIMUM_CONCATENATED_NAME_LENGTH: - return { - 'name_hash': None, - 'content_hash': None, - } + return {'name_hash': None, 'content_hash': None, 'dir_hash': None} + + # Concatenate the unique file names without the extensions, adding a space and sorting them alphabetically + unique_file_names_list = list(unique_file_names) + unique_file_names_list.sort() + concatenated_names = ' '.join(unique_file_names_list) + + # We do the same for the directory names, adding a space and sorting them alphabetically + unique_directories_list = list(unique_directories) + unique_directories_list.sort() + concatenated_directories = ' '.join(unique_directories_list) names_simhash = simhash(WordFeatureSet(concatenated_names.encode('utf-8'))) + dir_simhash = simhash(WordFeatureSet(concatenated_directories.encode('utf-8'))) content_simhash = fingerprint(vectorize_bytes(file_hashes)) - return { - 'name_hash': names_simhash, - 'content_hash': content_simhash, - } + return {'name_hash': names_simhash, 'content_hash': content_simhash, 'dir_hash': dir_simhash} - def present(self, output_format: str = None, output_file: str = None): + def present(self, output_format: Optional[str] = None, output_file: Optional[str] = None): """Present the hashed tree in the selected format""" self.presenter.present(output_format=output_format, output_file=output_file) From d571906f2fb893f983c8bec3e424e495ad49cc23 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 22 May 2025 11:59:18 +0200 Subject: [PATCH 350/489] [SP-2587] Update with papi request definition --- src/scanoss/scanners/folder_hasher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 9b59e837..601f6b51 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -199,7 +199,7 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: 'path_id': node.path, 'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None, 'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None, - 'sim_hash_dir': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None, + 'sim_hash_dir_names': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None, 'children': [self._hash_calc_from_node(child) for child in node.children.values()], } From 1632e0ae742846dfbcc9ffd3eeb54e78ac053c1e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 23 May 2025 14:34:32 +0200 Subject: [PATCH 351/489] [SP-2587] Use relative path for path_id --- src/scanoss/scanners/folder_hasher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 601f6b51..77bb454a 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -194,9 +194,10 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: dict: The computed hash data for the node. """ hash_data = self._hash_calc(node) + rel_path = Path(node.path).relative_to(self.scan_dir) return { - 'path_id': node.path, + 'path_id': str(rel_path), 'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None, 'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None, 'sim_hash_dir_names': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None, From 7302bd5c246376a6efd7025d0fd324bd0649074f Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 23 May 2025 14:38:08 +0200 Subject: [PATCH 352/489] [SP-2587] Pr comments --- src/scanoss/scanners/folder_hasher.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 77bb454a..46cb7530 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -234,10 +234,7 @@ def _hash_calc(self, node: DirectoryNode) -> dict: file_name_without_extension, _ = os.path.splitext(file_name) current_directory = os.path.dirname(file.path) - last_directory = os.path.basename(current_directory) - - if last_directory == '': - last_directory = os.path.basename(os.getcwd()) + last_directory = Path(current_directory).name or Path(self.scan_dir).name processed_hashes.add(key_str) unique_file_names.add(file_name_without_extension) From 3c8af3d5b01daa75213095331aaac0612e404619 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 16 Jun 2025 13:14:56 +0200 Subject: [PATCH 353/489] [SP-2587] feat: update folder_hasher to include language extensions + align with golang miner --- .../api/common/v2/scanoss_common_pb2.py | 53 ++++++++------ .../api/common/v2/scanoss_common_pb2_grpc.py | 20 ++++++ .../api/scanning/v2/scanoss_scanning_pb2.py | 71 ++++++++++++------- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 57 ++++++++++++--- src/scanoss/scanners/folder_hasher.py | 49 ++++++++++--- 5 files changed, 186 insertions(+), 64 deletions(-) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index cbcd2e5b..6ccec8ef 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/common/v2/scanoss-common.proto +# Protobuf Python Version: 5.27.2 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 5, + 27, + 2, + '', + 'scanoss/api/common/v2/scanoss-common.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -15,24 +26,24 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\")\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _STATUSCODE._serialized_start=379 - _STATUSCODE._serialized_end=475 - _STATUSRESPONSE._serialized_start=69 - _STATUSRESPONSE._serialized_end=153 - _ECHOREQUEST._serialized_start=155 - _ECHOREQUEST._serialized_end=185 - _ECHORESPONSE._serialized_start=187 - _ECHORESPONSE._serialized_end=218 - _PURLREQUEST._serialized_start=220 - _PURLREQUEST._serialized_end=334 - _PURLREQUEST_PURLS._serialized_start=292 - _PURLREQUEST_PURLS._serialized_end=334 - _PURL._serialized_start=336 - _PURL._serialized_end=377 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' + _globals['_STATUSCODE']._serialized_start=379 + _globals['_STATUSCODE']._serialized_end=475 + _globals['_STATUSRESPONSE']._serialized_start=69 + _globals['_STATUSRESPONSE']._serialized_end=153 + _globals['_ECHOREQUEST']._serialized_start=155 + _globals['_ECHOREQUEST']._serialized_end=185 + _globals['_ECHORESPONSE']._serialized_start=187 + _globals['_ECHORESPONSE']._serialized_end=218 + _globals['_PURLREQUEST']._serialized_start=220 + _globals['_PURLREQUEST']._serialized_end=334 + _globals['_PURLREQUEST_PURLS']._serialized_start=292 + _globals['_PURLREQUEST_PURLS']._serialized_end=334 + _globals['_PURL']._serialized_start=336 + _globals['_PURL']._serialized_end=377 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index 2daafffe..693dc2ea 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -1,4 +1,24 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings + +GRPC_GENERATED_VERSION = '1.67.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/common/v2/scanoss_common_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index b81bf7e9..700b2395 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/scanning/v2/scanoss-scanning.proto +# Protobuf Python Version: 5.27.2 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 5, + 27, + 2, + '', + 'scanoss/api/scanning/v2/scanoss-scanning.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -16,28 +27,40 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xff\x01\n\nHFHRequest\x12\x12\n\nbest_match\x18\x01 \x01(\x08\x12\x11\n\tthreshold\x18\x02 \x01(\x05\x12:\n\x04root\x18\x03 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x1a\x8d\x01\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\"\xc1\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x1a\x81\x01\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component\x12\x13\n\x0bprobability\x18\x03 \x01(\x02\x12\r\n\x05stage\x18\x04 \x01(\x05\x32\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xf4\x03\n\nHFHRequest\x12\x16\n\nbest_match\x18\x01 \x01(\x08\x42\x02\x18\x01\x12\x15\n\tthreshold\x18\x02 \x01(\x05\x42\x02\x18\x01\x12:\n\x04root\x18\x03 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x04 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x05 \x01(\t\x12\x13\n\x0bquery_limit\x18\x06 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\xda\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aJ\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x12\x0f\n\x07version\x18\x04 \x01(\t\x1a\x89\x01\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component\x12\x17\n\x0bprobability\x18\x03 \x01(\x02\x42\x02\x18\x01\x12\x11\n\x05stage\x18\x04 \x01(\x05\x42\x02\x18\x01\x32\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service\"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _SCANNING.methods_by_name['Echo']._options = None - _SCANNING.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' - _SCANNING.methods_by_name['FolderHashScan']._options = None - _SCANNING.methods_by_name['FolderHashScan']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/scanning/hfh/scan:\001*' - _HFHREQUEST._serialized_start=196 - _HFHREQUEST._serialized_end=451 - _HFHREQUEST_CHILDREN._serialized_start=310 - _HFHREQUEST_CHILDREN._serialized_end=451 - _HFHRESPONSE._serialized_start=454 - _HFHRESPONSE._serialized_end=775 - _HFHRESPONSE_COMPONENT._serialized_start=586 - _HFHRESPONSE_COMPONENT._serialized_end=643 - _HFHRESPONSE_RESULT._serialized_start=646 - _HFHRESPONSE_RESULT._serialized_end=775 - _SCANNING._serialized_start=778 - _SCANNING._serialized_end=1035 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.scanning.v2.scanoss_scanning_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service\"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._loaded_options = None + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_options = b'8\001' + _globals['_HFHREQUEST'].fields_by_name['best_match']._loaded_options = None + _globals['_HFHREQUEST'].fields_by_name['best_match']._serialized_options = b'\030\001' + _globals['_HFHREQUEST'].fields_by_name['threshold']._loaded_options = None + _globals['_HFHREQUEST'].fields_by_name['threshold']._serialized_options = b'\030\001' + _globals['_HFHRESPONSE_RESULT'].fields_by_name['probability']._loaded_options = None + _globals['_HFHRESPONSE_RESULT'].fields_by_name['probability']._serialized_options = b'\030\001' + _globals['_HFHRESPONSE_RESULT'].fields_by_name['stage']._loaded_options = None + _globals['_HFHRESPONSE_RESULT'].fields_by_name['stage']._serialized_options = b'\030\001' + _globals['_SCANNING'].methods_by_name['Echo']._loaded_options = None + _globals['_SCANNING'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' + _globals['_SCANNING'].methods_by_name['FolderHashScan']._loaded_options = None + _globals['_SCANNING'].methods_by_name['FolderHashScan']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/scanning/hfh/scan:\001*' + _globals['_HFHREQUEST']._serialized_start=196 + _globals['_HFHREQUEST']._serialized_end=696 + _globals['_HFHREQUEST_CHILDREN']._serialized_start=381 + _globals['_HFHREQUEST_CHILDREN']._serialized_end=696 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=643 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=696 + _globals['_HFHRESPONSE']._serialized_start=699 + _globals['_HFHRESPONSE']._serialized_end=1045 + _globals['_HFHRESPONSE_COMPONENT']._serialized_start=831 + _globals['_HFHRESPONSE_COMPONENT']._serialized_end=905 + _globals['_HFHRESPONSE_RESULT']._serialized_start=908 + _globals['_HFHRESPONSE_RESULT']._serialized_end=1045 + _globals['_SCANNING']._serialized_start=1048 + _globals['_SCANNING']._serialized_end=1305 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index d00b7e38..a90de4a1 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -1,10 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.scanning.v2 import scanoss_scanning_pb2 as scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2 +GRPC_GENERATED_VERSION = '1.67.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class ScanningStub(object): """* @@ -21,12 +41,12 @@ def __init__(self, channel): '/scanoss.api.scanning.v2.Scanning/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.FolderHashScan = channel.unary_unary( '/scanoss.api.scanning.v2.Scanning/FolderHashScan', request_serializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, - ) + _registered_method=True) class ScanningServicer(object): @@ -65,6 +85,7 @@ def add_ScanningServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.scanning.v2.Scanning', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.scanning.v2.Scanning', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -84,11 +105,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.scanning.v2.Scanning/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def FolderHashScan(request, @@ -101,8 +132,18 @@ def FolderHashScan(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/FolderHashScan', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.scanning.v2.Scanning/FolderHashScan', scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 46cb7530..099a1d86 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -15,7 +15,7 @@ MINIMUM_FILE_COUNT = 8 MINIMUM_CONCATENATED_NAME_LENGTH = 32 -MINIMUM_FILE_NAME_LENGTH = 32 +MINIMUM_FILE_NAME_LENGTH = 64 class DirectoryNode: @@ -185,7 +185,7 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: Recursively compute folder hash data for a directory node. The hash data includes the path identifier, simhash for file names, - simhash for file content, and children node hash information. + simhash for file content, directory hash, language extensions, and children node hash information. Args: node (DirectoryNode): The directory node to compute the hash for. @@ -194,13 +194,22 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: dict: The computed hash data for the node. """ hash_data = self._hash_calc(node) - rel_path = Path(node.path).relative_to(self.scan_dir) + + # Safely calculate relative path + try: + node_path = Path(node.path).resolve() + scan_dir_path = Path(self.scan_dir).resolve() + rel_path = node_path.relative_to(scan_dir_path) + except ValueError: + # If relative_to fails, use the node path as is or a fallback + rel_path = Path(node.path).name if node.path else Path('.') return { 'path_id': str(rel_path), 'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None, 'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None, 'sim_hash_dir_names': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None, + 'language_extensions': hash_data['language_extensions'], 'children': [self._hash_calc_from_node(child) for child in node.children.values()], } @@ -217,11 +226,12 @@ def _hash_calc(self, node: DirectoryNode) -> dict: node (DirectoryNode): The directory node containing file items. Returns: - dict: A dictionary with 'name_hash' and 'content_hash' keys. + dict: A dictionary with 'name_hash', 'content_hash', 'dir_hash', and 'language_extensions' keys. """ processed_hashes = set() unique_file_names = set() unique_directories = set() + extension_map = {} file_hashes = [] selected_names = [] @@ -231,25 +241,32 @@ def _hash_calc(self, node: DirectoryNode) -> dict: continue file_name = os.path.basename(file.path) - file_name_without_extension, _ = os.path.splitext(file_name) + + file_name_without_extension, extension = os.path.splitext(file_name) current_directory = os.path.dirname(file.path) - last_directory = Path(current_directory).name or Path(self.scan_dir).name + if extension and len(extension) > 1: + ext_without_dot = extension[1:] + extension_map[ext_without_dot] = extension_map.get(ext_without_dot, 0) + 1 + + if current_directory and current_directory != '.': + last_directory = os.path.basename(current_directory) + if last_directory != current_directory and last_directory not in ['.', '..']: + unique_directories.add(last_directory) processed_hashes.add(key_str) unique_file_names.add(file_name_without_extension) - unique_directories.add(last_directory) selected_names.append(file_name) file_hashes.append(file.key) if len(selected_names) < MINIMUM_FILE_COUNT: - return {'name_hash': None, 'content_hash': None, 'dir_hash': None} + return {'name_hash': None, 'content_hash': None, 'dir_hash': None, 'language_extensions': None} selected_names.sort() concatenated_names = ''.join(selected_names) if len(concatenated_names.encode('utf-8')) < MINIMUM_CONCATENATED_NAME_LENGTH: - return {'name_hash': None, 'content_hash': None, 'dir_hash': None} + return {'name_hash': None, 'content_hash': None, 'dir_hash': None, 'language_extensions': None} # Concatenate the unique file names without the extensions, adding a space and sorting them alphabetically unique_file_names_list = list(unique_file_names) @@ -257,7 +274,7 @@ def _hash_calc(self, node: DirectoryNode) -> dict: concatenated_names = ' '.join(unique_file_names_list) # We do the same for the directory names, adding a space and sorting them alphabetically - unique_directories_list = list(unique_directories) + unique_directories_list = [d for d in unique_directories if d not in ['.', '..']] unique_directories_list.sort() concatenated_directories = ' '.join(unique_directories_list) @@ -265,7 +282,17 @@ def _hash_calc(self, node: DirectoryNode) -> dict: dir_simhash = simhash(WordFeatureSet(concatenated_directories.encode('utf-8'))) content_simhash = fingerprint(vectorize_bytes(file_hashes)) - return {'name_hash': names_simhash, 'content_hash': content_simhash, 'dir_hash': dir_simhash} + # Debug logging similar to Go implementation + self.base.print_debug(f'Unique file names: {unique_file_names_list}') + self.base.print_debug(f'Unique directories: {unique_directories_list}') + self.base.print_debug(f'{dir_simhash:x}/{names_simhash:x} - {content_simhash:x} - {extension_map}') + + return { + 'name_hash': names_simhash, + 'content_hash': content_simhash, + 'dir_hash': dir_simhash, + 'language_extensions': extension_map, + } def present(self, output_format: Optional[str] = None, output_file: Optional[str] = None): """Present the hashed tree in the selected format""" From f4fe83fb510ba7d48e2071e1b7a345675dfddfbd Mon Sep 17 00:00:00 2001 From: coresoftware dev Date: Mon, 16 Jun 2025 14:50:14 +0200 Subject: [PATCH 354/489] fix dir hash --- src/scanoss/scanners/folder_hasher.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 099a1d86..1cc6ee86 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -209,7 +209,7 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: 'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None, 'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None, 'sim_hash_dir_names': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None, - 'language_extensions': hash_data['language_extensions'], + 'lang_extensions': hash_data['lang_extensions'], 'children': [self._hash_calc_from_node(child) for child in node.children.values()], } @@ -226,7 +226,7 @@ def _hash_calc(self, node: DirectoryNode) -> dict: node (DirectoryNode): The directory node containing file items. Returns: - dict: A dictionary with 'name_hash', 'content_hash', 'dir_hash', and 'language_extensions' keys. + dict: A dictionary with 'name_hash', 'content_hash', 'dir_hash', and 'lang_extensions' keys. """ processed_hashes = set() unique_file_names = set() @@ -249,24 +249,22 @@ def _hash_calc(self, node: DirectoryNode) -> dict: ext_without_dot = extension[1:] extension_map[ext_without_dot] = extension_map.get(ext_without_dot, 0) + 1 - if current_directory and current_directory != '.': - last_directory = os.path.basename(current_directory) - if last_directory != current_directory and last_directory not in ['.', '..']: - unique_directories.add(last_directory) + last_directory = Path(current_directory).name or Path(self.scan_dir).name processed_hashes.add(key_str) unique_file_names.add(file_name_without_extension) + unique_directories.add(last_directory) selected_names.append(file_name) file_hashes.append(file.key) if len(selected_names) < MINIMUM_FILE_COUNT: - return {'name_hash': None, 'content_hash': None, 'dir_hash': None, 'language_extensions': None} + return {'name_hash': None, 'content_hash': None, 'dir_hash': None, 'lang_extensions': None} selected_names.sort() concatenated_names = ''.join(selected_names) if len(concatenated_names.encode('utf-8')) < MINIMUM_CONCATENATED_NAME_LENGTH: - return {'name_hash': None, 'content_hash': None, 'dir_hash': None, 'language_extensions': None} + return {'name_hash': None, 'content_hash': None, 'dir_hash': None, 'lang_extensions': None} # Concatenate the unique file names without the extensions, adding a space and sorting them alphabetically unique_file_names_list = list(unique_file_names) @@ -274,7 +272,7 @@ def _hash_calc(self, node: DirectoryNode) -> dict: concatenated_names = ' '.join(unique_file_names_list) # We do the same for the directory names, adding a space and sorting them alphabetically - unique_directories_list = [d for d in unique_directories if d not in ['.', '..']] + unique_directories_list = list(unique_directories) unique_directories_list.sort() concatenated_directories = ' '.join(unique_directories_list) @@ -291,7 +289,7 @@ def _hash_calc(self, node: DirectoryNode) -> dict: 'name_hash': names_simhash, 'content_hash': content_simhash, 'dir_hash': dir_simhash, - 'language_extensions': extension_map, + 'lang_extensions': extension_map, } def present(self, output_format: Optional[str] = None, output_file: Optional[str] = None): From 1bd832dc200fa263f3346435874354881238f3db Mon Sep 17 00:00:00 2001 From: coresoftware dev Date: Thu, 19 Jun 2025 12:20:46 +0200 Subject: [PATCH 355/489] [SP-2587] feat: update scanner from new papi definition --- docs/source/index.rst | 4 -- .../api/common/v2/scanoss_common_pb2.py | 8 ++-- .../api/common/v2/scanoss_common_pb2_grpc.py | 2 +- .../api/scanning/v2/scanoss_scanning_pb2.py | 44 ++++++++----------- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 2 +- src/scanoss/cli.py | 18 -------- src/scanoss/scanners/scanner_hfh.py | 4 -- 7 files changed, 24 insertions(+), 58 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index f70f93d7..21c1947c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -252,10 +252,6 @@ Performs a comprehensive scan of a directory using folder hashing to identify co - Output format: {json} (optional - default json) * - --timeout , -M - Timeout in seconds for API communication (optional - default 600) - * - --best-match, -bm - - Enable best match mode (optional - default: False) - * - --threshold <1-100> - - Threshold for result matching (optional - default: 100) * - --settings , -st - Settings file to use for scanning (optional - default scanoss.json) * - --skip-settings-file, -stf diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index 6ccec8ef..e8ac6e13 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/common/v2/scanoss-common.proto -# Protobuf Python Version: 5.27.2 +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -11,9 +11,9 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, - 5, - 27, - 2, + 6, + 31, + 0, '', 'scanoss/api/common/v2/scanoss-common.proto' ) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index 693dc2ea..0fc410e3 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -4,7 +4,7 @@ import warnings -GRPC_GENERATED_VERSION = '1.67.0' +GRPC_GENERATED_VERSION = '1.73.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 700b2395..49dd2717 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/scanning/v2/scanoss-scanning.proto -# Protobuf Python Version: 5.27.2 +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -11,9 +11,9 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, - 5, - 27, - 2, + 6, + 31, + 0, '', 'scanoss/api/scanning/v2/scanoss-scanning.proto' ) @@ -27,7 +27,7 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xf4\x03\n\nHFHRequest\x12\x16\n\nbest_match\x18\x01 \x01(\x08\x42\x02\x18\x01\x12\x15\n\tthreshold\x18\x02 \x01(\x05\x42\x02\x18\x01\x12:\n\x04root\x18\x03 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x04 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x05 \x01(\t\x12\x13\n\x0bquery_limit\x18\x06 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\xda\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aJ\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x12\x0f\n\x07version\x18\x04 \x01(\t\x1a\x89\x01\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component\x12\x17\n\x0bprobability\x18\x03 \x01(\x02\x42\x02\x18\x01\x12\x11\n\x05stage\x18\x04 \x01(\x05\x42\x02\x18\x01\x32\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\x9c\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -37,30 +37,22 @@ _globals['DESCRIPTOR']._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\222A\323\001\022m\n\030SCANOSS Scanning Service\"L\n\020scanoss-scanning\022#https://github.com/scanoss/scanning\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._loaded_options = None _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_options = b'8\001' - _globals['_HFHREQUEST'].fields_by_name['best_match']._loaded_options = None - _globals['_HFHREQUEST'].fields_by_name['best_match']._serialized_options = b'\030\001' - _globals['_HFHREQUEST'].fields_by_name['threshold']._loaded_options = None - _globals['_HFHREQUEST'].fields_by_name['threshold']._serialized_options = b'\030\001' - _globals['_HFHRESPONSE_RESULT'].fields_by_name['probability']._loaded_options = None - _globals['_HFHRESPONSE_RESULT'].fields_by_name['probability']._serialized_options = b'\030\001' - _globals['_HFHRESPONSE_RESULT'].fields_by_name['stage']._loaded_options = None - _globals['_HFHRESPONSE_RESULT'].fields_by_name['stage']._serialized_options = b'\030\001' _globals['_SCANNING'].methods_by_name['Echo']._loaded_options = None _globals['_SCANNING'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' _globals['_SCANNING'].methods_by_name['FolderHashScan']._loaded_options = None _globals['_SCANNING'].methods_by_name['FolderHashScan']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/scanning/hfh/scan:\001*' _globals['_HFHREQUEST']._serialized_start=196 - _globals['_HFHREQUEST']._serialized_end=696 - _globals['_HFHREQUEST_CHILDREN']._serialized_start=381 - _globals['_HFHREQUEST_CHILDREN']._serialized_end=696 - _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=643 - _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=696 - _globals['_HFHRESPONSE']._serialized_start=699 - _globals['_HFHRESPONSE']._serialized_end=1045 - _globals['_HFHRESPONSE_COMPONENT']._serialized_start=831 - _globals['_HFHRESPONSE_COMPONENT']._serialized_end=905 - _globals['_HFHRESPONSE_RESULT']._serialized_start=908 - _globals['_HFHRESPONSE_RESULT']._serialized_end=1045 - _globals['_SCANNING']._serialized_start=1048 - _globals['_SCANNING']._serialized_end=1305 + _globals['_HFHREQUEST']._serialized_end=649 + _globals['_HFHREQUEST_CHILDREN']._serialized_start=334 + _globals['_HFHREQUEST_CHILDREN']._serialized_end=649 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=596 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=649 + _globals['_HFHRESPONSE']._serialized_start=652 + _globals['_HFHRESPONSE']._serialized_end=936 + _globals['_HFHRESPONSE_COMPONENT']._serialized_start=784 + _globals['_HFHRESPONSE_COMPONENT']._serialized_end=841 + _globals['_HFHRESPONSE_RESULT']._serialized_start=843 + _globals['_HFHRESPONSE_RESULT']._serialized_end=936 + _globals['_SCANNING']._serialized_start=939 + _globals['_SCANNING']._serialized_end=1196 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index a90de4a1..06d71530 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -6,7 +6,7 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.scanning.v2 import scanoss_scanning_pb2 as scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2 -GRPC_GENERATED_VERSION = '1.67.0' +GRPC_GENERATED_VERSION = '1.73.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 418a68e8..a1cdb35a 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -627,21 +627,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 default='json', help='Result output format (optional - default: json)', ) - p_folder_scan.add_argument( - '--best-match', - '-bm', - action='store_true', - default=False, - help='Enable best match mode (optional - default: False)', - ) - p_folder_scan.add_argument( - '--threshold', - type=int, - choices=range(1, 101), - metavar='1-100', - default=100, - help='Threshold for result matching (optional - default: 100)', - ) p_folder_scan.set_defaults(func=folder_hashing_scan) # Sub-command: folder-hash @@ -1967,9 +1952,6 @@ def folder_hashing_scan(parser, args): scanoss_settings=scanoss_settings, ) - scanner.best_match = args.best_match - scanner.threshold = args.threshold - if scanner.scan(): scanner.present(output_file=args.output, output_format=args.format) except ScanossGrpcError as e: diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 4b573845..af52b455 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -88,8 +88,6 @@ def __init__( self.scan_dir = scan_dir self.client = client self.scan_results = None - self.best_match = False - self.threshold = 100 def scan(self) -> Optional[Dict]: """ @@ -100,8 +98,6 @@ def scan(self) -> Optional[Dict]: """ hfh_request = { 'root': self.folder_hasher.hash_directory(self.scan_dir), - 'threshold': self.threshold, - 'best_match': self.best_match, } spinner = Spinner('Scanning folder...') From 738ac6399c364511ad1cb248907f81fe98443513 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 19 Jun 2025 12:36:24 +0200 Subject: [PATCH 356/489] [SP-2587] feat: use older version of protobuf generation --- src/scanoss/api/common/v2/scanoss_common_pb2.py | 8 ++++---- src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py | 2 +- src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py | 8 ++++---- src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index e8ac6e13..6ccec8ef 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/common/v2/scanoss-common.proto -# Protobuf Python Version: 6.31.0 +# Protobuf Python Version: 5.27.2 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -11,9 +11,9 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, - 6, - 31, - 0, + 5, + 27, + 2, '', 'scanoss/api/common/v2/scanoss-common.proto' ) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index 0fc410e3..693dc2ea 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -4,7 +4,7 @@ import warnings -GRPC_GENERATED_VERSION = '1.73.0' +GRPC_GENERATED_VERSION = '1.67.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 49dd2717..1d395208 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/scanning/v2/scanoss-scanning.proto -# Protobuf Python Version: 6.31.0 +# Protobuf Python Version: 5.27.2 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -11,9 +11,9 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, - 6, - 31, - 0, + 5, + 27, + 2, '', 'scanoss/api/scanning/v2/scanoss-scanning.proto' ) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index 06d71530..a90de4a1 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -6,7 +6,7 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.scanning.v2 import scanoss_scanning_pb2 as scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2 -GRPC_GENERATED_VERSION = '1.73.0' +GRPC_GENERATED_VERSION = '1.67.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False From 016dfc2caa3ff0085d2dbd3ad84240e34e961f37 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 26 Jun 2025 09:49:00 +0200 Subject: [PATCH 357/489] feat: add rank_threshold to folder hashing request --- .../api/scanning/v2/scanoss_scanning_pb2.py | 18 ++++++++++-------- src/scanoss/cli.py | 8 ++++++++ src/scanoss/scanners/scanner_hfh.py | 4 ++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 1d395208..3e1ae6ba 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\x9c\x02\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\x84\x03\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a)\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x1av\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12>\n\x08versions\x18\x02 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHResponse.Version\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x12\r\n\x05order\x18\x04 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -48,11 +48,13 @@ _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=596 _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=649 _globals['_HFHRESPONSE']._serialized_start=652 - _globals['_HFHRESPONSE']._serialized_end=936 - _globals['_HFHRESPONSE_COMPONENT']._serialized_start=784 - _globals['_HFHRESPONSE_COMPONENT']._serialized_end=841 - _globals['_HFHRESPONSE_RESULT']._serialized_start=843 - _globals['_HFHRESPONSE_RESULT']._serialized_end=936 - _globals['_SCANNING']._serialized_start=939 - _globals['_SCANNING']._serialized_end=1196 + _globals['_HFHRESPONSE']._serialized_end=1040 + _globals['_HFHRESPONSE_VERSION']._serialized_start=784 + _globals['_HFHRESPONSE_VERSION']._serialized_end=825 + _globals['_HFHRESPONSE_COMPONENT']._serialized_start=827 + _globals['_HFHRESPONSE_COMPONENT']._serialized_end=945 + _globals['_HFHRESPONSE_RESULT']._serialized_start=947 + _globals['_HFHRESPONSE_RESULT']._serialized_end=1040 + _globals['_SCANNING']._serialized_start=1043 + _globals['_SCANNING']._serialized_end=1300 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index a1cdb35a..ae3fcd68 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -627,6 +627,13 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 default='json', help='Result output format (optional - default: json)', ) + p_folder_scan.add_argument( + '--rank-threshold', + type=int, + default=9, + help='Get results with rank below this threshold (e.g i only want to see results from rank 9 and below). ' + 'Lower rank means better quality.', + ) p_folder_scan.set_defaults(func=folder_hashing_scan) # Sub-command: folder-hash @@ -1950,6 +1957,7 @@ def folder_hashing_scan(parser, args): config=scanner_config, client=client, scanoss_settings=scanoss_settings, + rank_threshold=args.rank_threshold, ) if scanner.scan(): diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index af52b455..48bb4874 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -52,6 +52,7 @@ def __init__( config: ScannerConfig, client: Optional[ScanossGrpc] = None, scanoss_settings: Optional[ScanossSettings] = None, + rank_threshold: int = 9, ): """ Initialize the ScannerHFH. @@ -61,6 +62,7 @@ def __init__( config (ScannerConfig): Configuration parameters for the scanner. client (ScanossGrpc): gRPC client for communicating with the scanning service. scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss. + rank_threshold (int): Get results with rank below this threshold (default: 9). """ self.base = ScanossBase( debug=config.debug, @@ -88,6 +90,7 @@ def __init__( self.scan_dir = scan_dir self.client = client self.scan_results = None + self.rank_threshold = rank_threshold def scan(self) -> Optional[Dict]: """ @@ -98,6 +101,7 @@ def scan(self) -> Optional[Dict]: """ hfh_request = { 'root': self.folder_hasher.hash_directory(self.scan_dir), + 'rank_threshold': self.rank_threshold, } spinner = Spinner('Scanning folder...') From b3b78e334aa156ca8076d9491df6dc5ceb5bab20 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 30 Jun 2025 13:39:05 +0200 Subject: [PATCH 358/489] [SP-2587] feat: add cyclonedx format to hfh --- .../api/scanning/v2/scanoss_scanning_pb2.py | 16 ++++---- src/scanoss/cli.py | 7 ++-- src/scanoss/constants.py | 2 + src/scanoss/scanners/scanner_hfh.py | 38 +++++++++++++++++-- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 3e1ae6ba..c88a2ded 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\x84\x03\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a)\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x1av\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12>\n\x08versions\x18\x02 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHResponse.Version\x12\x0c\n\x04rank\x18\x03 \x01(\x05\x12\r\n\x05order\x18\x04 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\xa3\x03\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a)\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x1a\x94\x01\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0e\n\x06vendor\x18\x03 \x01(\t\x12>\n\x08versions\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHResponse.Version\x12\x0c\n\x04rank\x18\x05 \x01(\x05\x12\r\n\x05order\x18\x06 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -48,13 +48,13 @@ _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=596 _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=649 _globals['_HFHRESPONSE']._serialized_start=652 - _globals['_HFHRESPONSE']._serialized_end=1040 + _globals['_HFHRESPONSE']._serialized_end=1071 _globals['_HFHRESPONSE_VERSION']._serialized_start=784 _globals['_HFHRESPONSE_VERSION']._serialized_end=825 - _globals['_HFHRESPONSE_COMPONENT']._serialized_start=827 - _globals['_HFHRESPONSE_COMPONENT']._serialized_end=945 - _globals['_HFHRESPONSE_RESULT']._serialized_start=947 - _globals['_HFHRESPONSE_RESULT']._serialized_end=1040 - _globals['_SCANNING']._serialized_start=1043 - _globals['_SCANNING']._serialized_end=1300 + _globals['_HFHRESPONSE_COMPONENT']._serialized_start=828 + _globals['_HFHRESPONSE_COMPONENT']._serialized_end=976 + _globals['_HFHRESPONSE_RESULT']._serialized_start=978 + _globals['_HFHRESPONSE_RESULT']._serialized_end=1071 + _globals['_SCANNING']._serialized_start=1074 + _globals['_SCANNING']._serialized_end=1331 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index ae3fcd68..8eedd45c 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -54,6 +54,7 @@ from .components import Components from .constants import ( DEFAULT_API_TIMEOUT, + DEFAULT_HFH_RANK_THRESHOLD, DEFAULT_POST_SIZE, DEFAULT_RETRY, DEFAULT_TIMEOUT, @@ -623,15 +624,15 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--format', '-f', type=str, - choices=['json'], + choices=['json', 'cyclonedx'], default='json', help='Result output format (optional - default: json)', ) p_folder_scan.add_argument( '--rank-threshold', type=int, - default=9, - help='Get results with rank below this threshold (e.g i only want to see results from rank 9 and below). ' + default=DEFAULT_HFH_RANK_THRESHOLD, + help='Get results with rank below this threshold (e.g i only want to see results from rank 5 and below). ' 'Lower rank means better quality.', ) p_folder_scan.set_defaults(func=folder_hashing_scan) diff --git a/src/scanoss/constants.py b/src/scanoss/constants.py index 1dd9bd61..92fc15b7 100644 --- a/src/scanoss/constants.py +++ b/src/scanoss/constants.py @@ -12,3 +12,5 @@ DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL DEFAULT_API_TIMEOUT = 600 + +DEFAULT_HFH_RANK_THRESHOLD = 5 \ No newline at end of file diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 48bb4874..5e778055 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -29,6 +29,8 @@ from progress.spinner import Spinner +from scanoss.constants import DEFAULT_HFH_RANK_THRESHOLD +from scanoss.cyclonedx import CycloneDx from scanoss.file_filters import FileFilters from scanoss.scanners.folder_hasher import FolderHasher from scanoss.scanners.scanner_config import ScannerConfig @@ -52,7 +54,7 @@ def __init__( config: ScannerConfig, client: Optional[ScanossGrpc] = None, scanoss_settings: Optional[ScanossSettings] = None, - rank_threshold: int = 9, + rank_threshold: int = DEFAULT_HFH_RANK_THRESHOLD, ): """ Initialize the ScannerHFH. @@ -62,7 +64,7 @@ def __init__( config (ScannerConfig): Configuration parameters for the scanner. client (ScanossGrpc): gRPC client for communicating with the scanning service. scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss. - rank_threshold (int): Get results with rank below this threshold (default: 9). + rank_threshold (int): Get results with rank below this threshold (default: 5). """ self.base = ScanossBase( debug=config.debug, @@ -161,7 +163,37 @@ def _format_plain_output(self) -> str: ) def _format_cyclonedx_output(self) -> str: - raise NotImplementedError('CycloneDX output is not implemented') + if not self.scanner.scan_results: + return None + + try: + first_result = self.scanner.scan_results['results'][0] + best_match_component = [c for c in first_result['components'] if c['order'] == 1][0] + best_match_version = best_match_component['versions'][0] + purl = best_match_component['purl'] + + get_dependencies_json_request = { + 'files': [ + { + 'file': f'{best_match_component["name"]}:{best_match_version["version"]}', + 'purls': [{'purl': purl, 'requirement': best_match_version['version']}], + } + ] + } + + decorated_scan_results = self.scanner.client.get_dependencies(get_dependencies_json_request) + + cdx = CycloneDx(self.base.debug, self.output_file) + scan_results = {} + for f in decorated_scan_results['files']: + scan_results[f['file']] = [f] + if not cdx.produce_from_json(scan_results, self.output_file): + error_msg = 'ERROR: Failed to produce CycloneDX output' + self.base.print_stderr(error_msg) + raise ValueError(error_msg) + except Exception as e: + self.base.print_stderr(f'ERROR: Failed to get license information: {e}') + return None def _format_spdxlite_output(self) -> str: raise NotImplementedError('SPDXlite output is not implemented') From a59998a1a5a28d1e513bcff0b7722b8bbc9d9da5 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Fri, 27 Jun 2025 13:48:32 +0100 Subject: [PATCH 359/489] grpc stubs with python 2.8 --- .../api/common/v2/scanoss_common_pb2.py | 53 +++++++---------- .../api/common/v2/scanoss_common_pb2_grpc.py | 20 ------- .../api/scanning/v2/scanoss_scanning_pb2.py | 13 +---- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 57 +++---------------- 4 files changed, 30 insertions(+), 113 deletions(-) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index 6ccec8ef..cbcd2e5b 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -1,22 +1,11 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/common/v2/scanoss-common.proto -# Protobuf Python Version: 5.27.2 """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 5, - 27, - 2, - '', - 'scanoss/api/common/v2/scanoss-common.proto' -) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -26,24 +15,24 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\")\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _globals['_STATUSCODE']._serialized_start=379 - _globals['_STATUSCODE']._serialized_end=475 - _globals['_STATUSRESPONSE']._serialized_start=69 - _globals['_STATUSRESPONSE']._serialized_end=153 - _globals['_ECHOREQUEST']._serialized_start=155 - _globals['_ECHOREQUEST']._serialized_end=185 - _globals['_ECHORESPONSE']._serialized_start=187 - _globals['_ECHORESPONSE']._serialized_end=218 - _globals['_PURLREQUEST']._serialized_start=220 - _globals['_PURLREQUEST']._serialized_end=334 - _globals['_PURLREQUEST_PURLS']._serialized_start=292 - _globals['_PURLREQUEST_PURLS']._serialized_end=334 - _globals['_PURL']._serialized_start=336 - _globals['_PURL']._serialized_end=377 +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' + _STATUSCODE._serialized_start=379 + _STATUSCODE._serialized_end=475 + _STATUSRESPONSE._serialized_start=69 + _STATUSRESPONSE._serialized_end=153 + _ECHOREQUEST._serialized_start=155 + _ECHOREQUEST._serialized_end=185 + _ECHORESPONSE._serialized_start=187 + _ECHORESPONSE._serialized_end=218 + _PURLREQUEST._serialized_start=220 + _PURLREQUEST._serialized_end=334 + _PURLREQUEST_PURLS._serialized_start=292 + _PURLREQUEST_PURLS._serialized_end=334 + _PURL._serialized_start=336 + _PURL._serialized_end=377 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index 693dc2ea..2daafffe 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -1,24 +1,4 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc -import warnings - -GRPC_GENERATED_VERSION = '1.67.0' -GRPC_VERSION = grpc.__version__ -_version_not_supported = False - -try: - from grpc._utilities import first_version_is_lower - _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) -except ImportError: - _version_not_supported = True - -if _version_not_supported: - raise RuntimeError( - f'The grpc package installed is at version {GRPC_VERSION},' - + f' but the generated code in scanoss/api/common/v2/scanoss_common_pb2_grpc.py depends on' - + f' grpcio>={GRPC_GENERATED_VERSION}.' - + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' - + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' - ) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index c88a2ded..114a0bf8 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -1,22 +1,11 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/scanning/v2/scanoss-scanning.proto -# Protobuf Python Version: 5.27.2 """Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -_runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 5, - 27, - 2, - '', - 'scanoss/api/scanning/v2/scanoss-scanning.proto' -) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index a90de4a1..d00b7e38 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -1,30 +1,10 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc -import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.scanning.v2 import scanoss_scanning_pb2 as scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2 -GRPC_GENERATED_VERSION = '1.67.0' -GRPC_VERSION = grpc.__version__ -_version_not_supported = False - -try: - from grpc._utilities import first_version_is_lower - _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) -except ImportError: - _version_not_supported = True - -if _version_not_supported: - raise RuntimeError( - f'The grpc package installed is at version {GRPC_VERSION},' - + f' but the generated code in scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py depends on' - + f' grpcio>={GRPC_GENERATED_VERSION}.' - + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' - + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' - ) - class ScanningStub(object): """* @@ -41,12 +21,12 @@ def __init__(self, channel): '/scanoss.api.scanning.v2.Scanning/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - _registered_method=True) + ) self.FolderHashScan = channel.unary_unary( '/scanoss.api.scanning.v2.Scanning/FolderHashScan', request_serializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, - _registered_method=True) + ) class ScanningServicer(object): @@ -85,7 +65,6 @@ def add_ScanningServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.scanning.v2.Scanning', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) - server.add_registered_method_handlers('scanoss.api.scanning.v2.Scanning', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -105,21 +84,11 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/scanoss.api.scanning.v2.Scanning/Echo', + return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod def FolderHashScan(request, @@ -132,18 +101,8 @@ def FolderHashScan(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/scanoss.api.scanning.v2.Scanning/FolderHashScan', + return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/FolderHashScan', scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From 01026d09cacd4953e87f49c7d1cb75734082565e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 30 Jun 2025 14:06:36 +0200 Subject: [PATCH 360/489] [SP-2587] feat: improvements --- src/scanoss/scanners/scanner_hfh.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 5e778055..092003d0 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -164,11 +164,24 @@ def _format_plain_output(self) -> str: def _format_cyclonedx_output(self) -> str: if not self.scanner.scan_results: - return None - + return '' try: + if 'results' not in self.scanner.scan_results or not self.scanner.scan_results['results']: + self.base.print_stderr('ERROR: No scan results found') + return '' + first_result = self.scanner.scan_results['results'][0] - best_match_component = [c for c in first_result['components'] if c['order'] == 1][0] + + best_match_components = [c for c in first_result.get('components', []) if c.get('order') == 1] + if not best_match_components: + self.base.print_stderr('ERROR: No best match component found') + return '' + + best_match_component = best_match_components[0] + if not best_match_component.get('versions'): + self.base.print_stderr('ERROR: No versions found for best match component') + return '' + best_match_version = best_match_component['versions'][0] purl = best_match_component['purl'] From 9af796511780815c4b0dcfcef99a7a00739134fd Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 30 Jun 2025 16:12:12 +0200 Subject: [PATCH 361/489] [SP-2587] chore: update documentation --- CLIENT_HELP.md | 9 +++++++++ docs/source/index.rst | 4 +++- src/scanoss/cli.py | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 16e6aef0..12e2f164 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -485,6 +485,15 @@ The new `folder-scan` subcommand performs a comprehensive scan on an entire dire scanoss-py folder-scan /path/to/folder -o folder-scan-results.json ``` +**Options:** +- `--rank-threshold`: Filter results to only show those with rank value at or below this threshold (e.g., `--rank-threshold 3` returns results with rank 1, 2, or 3). Lower rank values indicate higher quality matches. +- `--format`: Result output format (json or cyclonedx, default: json) + +**Example with rank threshold:** +```shell +scanoss-py folder-scan /path/to/folder --rank-threshold 3 -o folder-scan-results.json +``` + ### Container-Scan a Docker Image The `container-scan` subcommand allows you to scan Docker container images for dependencies. This command extracts and analyzes dependencies from container images, helping you identify open source components within containerized applications. diff --git a/docs/source/index.rst b/docs/source/index.rst index 21c1947c..eb6a98f8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -249,9 +249,11 @@ Performs a comprehensive scan of a directory using folder hashing to identify co * - --output , -o - Output result file name (optional - default STDOUT) * - --format , -f - - Output format: {json} (optional - default json) + - Output format: {json, cyclonedx} (optional - default json) * - --timeout , -M - Timeout in seconds for API communication (optional - default 600) + * - --rank-threshold + - Filter results to only show those with rank value at or below this threshold (e.g., --rank-threshold 3 returns results with rank 1, 2, or 3). Lower rank values indicate higher quality matches. * - --settings , -st - Settings file to use for scanning (optional - default scanoss.json) * - --skip-settings-file, -stf diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 8eedd45c..9f7005cc 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -632,8 +632,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--rank-threshold', type=int, default=DEFAULT_HFH_RANK_THRESHOLD, - help='Get results with rank below this threshold (e.g i only want to see results from rank 5 and below). ' - 'Lower rank means better quality.', + help='Filter results to only show those with rank value at or below this threshold (e.g., --rank-threshold 3 returns results with rank 1, 2, or 3). ' + 'Lower rank values indicate higher quality matches.', ) p_folder_scan.set_defaults(func=folder_hashing_scan) From 2933009b1996d84a70cee9f90de72a3bce131bc0 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 30 Jun 2025 16:58:50 +0200 Subject: [PATCH 362/489] [SP-2587] fix: modify dir hash creation --- src/scanoss/scanners/folder_hasher.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 1cc6ee86..78f54c13 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -15,7 +15,7 @@ MINIMUM_FILE_COUNT = 8 MINIMUM_CONCATENATED_NAME_LENGTH = 32 -MINIMUM_FILE_NAME_LENGTH = 64 +MAXIMUM_FILE_NAME_LENGTH = 32 class DirectoryNode: @@ -140,7 +140,7 @@ def _build_root_node(self, path: str) -> DirectoryNode: root_node = DirectoryNode(str(root)) all_files = [ - f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) <= MINIMUM_FILE_NAME_LENGTH + f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) <= MAXIMUM_FILE_NAME_LENGTH ] filtered_files = self.file_filters.get_filtered_files_from_files(all_files, str(root)) @@ -249,11 +249,15 @@ def _hash_calc(self, node: DirectoryNode) -> dict: ext_without_dot = extension[1:] extension_map[ext_without_dot] = extension_map.get(ext_without_dot, 0) + 1 - last_directory = Path(current_directory).name or Path(self.scan_dir).name + current_directory.removeprefix(self.scan_dir) + parts = current_directory.split(os.path.sep) + for d in parts: + if d in {'', '.', '..'}: + continue + unique_directories.add(d) processed_hashes.add(key_str) unique_file_names.add(file_name_without_extension) - unique_directories.add(last_directory) selected_names.append(file_name) file_hashes.append(file.key) From 21def1b47c02916fdebcbede9f2d05d0a76bd658 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 30 Jun 2025 18:28:29 +0200 Subject: [PATCH 363/489] [SP-2587] chore: update version and changelog --- CHANGELOG.md | 8 +++++++- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 22 +++++++++++----------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 113cbca6..e14cfc90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.27.0] - 2025-06-30 +### Added +- Add directory hash calculation to folder hasher +- Add rank-threshold option to folder scan command + ## [1.26.3] - 2025-06-26 ### Fixed - Fixed crash in inspect subcommand when processing components that lack license information @@ -570,4 +575,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.26.0]: https://github.com/scanoss/scanoss.py/compare/v1.25.2...v1.26.0 [1.26.1]: https://github.com/scanoss/scanoss.py/compare/v1.26.0...v1.26.1 [1.26.2]: https://github.com/scanoss/scanoss.py/compare/v1.26.1...v1.26.2 -[1.26.3]: https://github.com/scanoss/scanoss.py/compare/v1.26.2...v1.26.3 \ No newline at end of file +[1.26.3]: https://github.com/scanoss/scanoss.py/compare/v1.26.2...v1.26.3 +[1.27.0]: https://github.com/scanoss/scanoss.py/compare/v1.26.3...v1.27.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 55786eb0..b6b742b4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.26.3' +__version__ = '1.27.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 9f7005cc..fc618eb4 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -632,8 +632,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--rank-threshold', type=int, default=DEFAULT_HFH_RANK_THRESHOLD, - help='Filter results to only show those with rank value at or below this threshold (e.g., --rank-threshold 3 returns results with rank 1, 2, or 3). ' - 'Lower rank values indicate higher quality matches.', + help='Filter results to only show those with rank value at or below this threshold (e.g., --rank-threshold 3 ' + 'returns results with rank 1, 2, or 3). Lower rank values indicate higher quality matches.', ) p_folder_scan.set_defaults(func=folder_hashing_scan) @@ -1448,7 +1448,7 @@ def utils_certloc(*_): Run the "utils certloc" sub-command :param _: ignored/unused """ - import certifi # noqa: PLC0415,I001 + import certifi # noqa: PLC0415,I001 print(f'CA Cert File: {certifi.where()}') @@ -1459,11 +1459,11 @@ def utils_cert_download(_, args): # pylint: disable=PLR0912 # noqa: PLR0912 :param _: ignore/unused :param args: Parsed arguments """ - import socket # noqa: PLC0415,I001 - import traceback # noqa: PLC0415,I001 - from urllib.parse import urlparse # noqa: PLC0415,I001 + import socket # noqa: PLC0415,I001 + import traceback # noqa: PLC0415,I001 + from urllib.parse import urlparse # noqa: PLC0415,I001 - from OpenSSL import SSL, crypto # noqa: PLC0415,I001 + from OpenSSL import SSL, crypto # noqa: PLC0415,I001 file = sys.stdout if args.output: @@ -1511,7 +1511,7 @@ def utils_pac_proxy(_, args): :param _: ignore/unused :param args: Parsed arguments """ - from pypac.resolver import ProxyResolver # noqa: PLC0415,I001 + from pypac.resolver import ProxyResolver # noqa: PLC0415,I001 if not args.pac: print_stderr('Error: No pac file option specified.') @@ -1585,7 +1585,7 @@ def crypto_algorithms(parser, args): sys.exit(1) except Exception as e: if args.debug: - import traceback # noqa: PLC0415,I001 + import traceback # noqa: PLC0415,I001 traceback.print_exc() print_stderr(f'ERROR: {e}') @@ -1627,7 +1627,7 @@ def crypto_hints(parser, args): sys.exit(1) except Exception as e: if args.debug: - import traceback # noqa: PLC0415,I001 + import traceback # noqa: PLC0415,I001 traceback.print_exc() print_stderr(f'ERROR: {e}') @@ -1669,7 +1669,7 @@ def crypto_versions_in_range(parser, args): sys.exit(1) except Exception as e: if args.debug: - import traceback # noqa: PLC0415,I001 + import traceback # noqa: PLC0415,I001 traceback.print_exc() print_stderr(f'ERROR: {e}') From 628eeca5fc271e2efa99e002b3748b23efeca551 Mon Sep 17 00:00:00 2001 From: coresoftware dev Date: Tue, 1 Jul 2025 03:15:28 +0200 Subject: [PATCH 364/489] python 3.8 backward compatibility change. Update folder hashing file filter --- src/scanoss/file_filters.py | 1 - src/scanoss/scanners/folder_hasher.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index d52ea386..0b3c382a 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -44,7 +44,6 @@ 'license.txt', 'license.md', 'copying.lib', - 'makefile', } DEFAULT_SKIPPED_FILES_HFH = { diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 78f54c13..ad1bad32 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -249,7 +249,7 @@ def _hash_calc(self, node: DirectoryNode) -> dict: ext_without_dot = extension[1:] extension_map[ext_without_dot] = extension_map.get(ext_without_dot, 0) + 1 - current_directory.removeprefix(self.scan_dir) + current_directory.replace(self.scan_dir, '', 1).lstrip(os.path.sep) parts = current_directory.split(os.path.sep) for d in parts: if d in {'', '.', '..'}: From e8f040a73bac09b3be31736dabc761e5db295b28 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 1 Jul 2025 09:05:21 +0200 Subject: [PATCH 365/489] [SP-2587] fix: add makefile to file filters for regular scan --- src/scanoss/file_filters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index 0b3c382a..d52ea386 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -44,6 +44,7 @@ 'license.txt', 'license.md', 'copying.lib', + 'makefile', } DEFAULT_SKIPPED_FILES_HFH = { From 05e762fc0da37e6e05d1f8932522902a1d2cec1a Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 9 Jul 2025 10:13:15 +0200 Subject: [PATCH 366/489] [SP-2870] fix: cyclonedx format output not writing to file --- CHANGELOG.md | 6 +++ src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 22 ++++++----- src/scanoss/scanner.py | 46 +++++++++++------------ src/scanoss/scanners/container_scanner.py | 6 ++- src/scanoss/scanners/scanner_hfh.py | 8 ++-- 6 files changed, 52 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e14cfc90..23578873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.27.1] - 2025-07-09 +### Fixed +- Fixed when running `folder-scan` with `--format cyclonedx` the output was not writing to file +- Fixed when running `container-scan` with `--format cyclonedx` the output was not writing to file + ## [1.27.0] - 2025-06-30 ### Added - Add directory hash calculation to folder hasher @@ -577,3 +582,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.26.2]: https://github.com/scanoss/scanoss.py/compare/v1.26.1...v1.26.2 [1.26.3]: https://github.com/scanoss/scanoss.py/compare/v1.26.2...v1.26.3 [1.27.0]: https://github.com/scanoss/scanoss.py/compare/v1.26.3...v1.27.0 +[1.27.1]: https://github.com/scanoss/scanoss.py/compare/v1.27.0...v1.27.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b6b742b4..cc543c19 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.27.0' +__version__ = '1.27.1' diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 4e4ebba2..4bf326b4 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -22,14 +22,13 @@ THE SOFTWARE. """ +import datetime import json import os.path import sys import uuid -import datetime from . import __version__ - from .scanossbase import ScanossBase from .spdxlite import SpdxLite @@ -171,17 +170,22 @@ def produce_from_file(self, json_file: str, output_file: str = None) -> bool: success = self.produce_from_str(f.read(), output_file) return success - def produce_from_json(self, data: json, output_file: str = None) -> bool: + def produce_from_json(self, data: json, output_file: str = None) -> tuple[bool, json]: # noqa: PLR0912 """ - Produce the CycloneDX output from the input data - :param data: JSON object - :param output_file: Output file (optional) - :return: True if successful, False otherwise + Produce the CycloneDX output from the raw scan results input data + + Args: + data (json): JSON object + output_file (str, optional): Output file (optional). Defaults to None. + + Returns: + bool: True if successful, False otherwise + json: The CycloneDX output """ cdx, vdx = self.parse(data) if not cdx: self.print_stderr('ERROR: No CycloneDX data returned for the JSON string provided.') - return False + return False, None self._spdx.load_license_data() # Load SPDX license name data for later reference # # Using CDX version 1.4: https://cyclonedx.org/docs/1.4/json/ @@ -264,7 +268,7 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: if output_file: file.close() - return True + return True, data def produce_from_str(self, json_str: str, output_file: str = None) -> bool: """ diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 4b59903e..76767ce7 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -22,40 +22,40 @@ THE SOFTWARE. """ +import datetime import json import os -from pathlib import Path import sys -import datetime +from pathlib import Path from typing import Any, Dict, List, Optional -import importlib_resources +import importlib_resources from progress.bar import Bar from progress.spinner import Spinner from pypac.parser import PACFile from scanoss.file_filters import FileFilters -from .scanossapi import ScanossApi -from .cyclonedx import CycloneDx -from .spdxlite import SpdxLite +from . import __version__ from .csvoutput import CsvOutput -from .threadedscanning import ThreadedScanning +from .cyclonedx import CycloneDx from .scancodedeps import ScancodeDeps -from .threadeddependencies import ThreadedDependencies, SCOPE -from .scanossgrpc import ScanossGrpc -from .scantype import ScanType -from .scanossbase import ScanossBase from .scanoss_settings import ScanossSettings +from .scanossapi import ScanossApi +from .scanossbase import ScanossBase +from .scanossgrpc import ScanossGrpc from .scanpostprocessor import ScanPostProcessor -from . import __version__ +from .scantype import ScanType +from .spdxlite import SpdxLite +from .threadeddependencies import SCOPE, ThreadedDependencies +from .threadedscanning import ThreadedScanning FAST_WINNOWING = False try: from scanoss_winnowing.winnowing import Winnowing FAST_WINNOWING = True -except ModuleNotFoundError or ImportError: +except (ModuleNotFoundError, ImportError): FAST_WINNOWING = False from .winnowing import Winnowing @@ -284,7 +284,7 @@ def is_dependency_scan(self): return True return False - def scan_folder_with_options( + def scan_folder_with_options( # noqa: PLR0913 self, scan_dir: str, deps_file: str = None, @@ -332,7 +332,7 @@ def scan_folder_with_options( success = False return success - def scan_folder(self, scan_dir: str) -> bool: + def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 """ Scan the specified folder producing fingerprints, send to the SCANOSS API and return results @@ -400,7 +400,7 @@ def scan_folder(self, scan_dir: str) -> bool: scan_block += wfp scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -484,7 +484,7 @@ def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> b self.__log_result(json.dumps(results, indent=2, sort_keys=True)) elif self.output_format == 'cyclonedx': cdx = CycloneDx(self.debug, self.scan_output) - success = cdx.produce_from_json(results) + success, _ = cdx.produce_from_json(results) elif self.output_format == 'spdxlite': spdxlite = SpdxLite(self.debug, self.scan_output) success = spdxlite.produce_from_json(results) @@ -509,7 +509,7 @@ def _merge_scan_results( for response in scan_responses: if response is not None: if file_map: - response = self._deobfuscate_filenames(response, file_map) + response = self._deobfuscate_filenames(response, file_map) # noqa: PLW2901 results.update(response) dep_files = dep_responses.get('files', None) if dep_responses else None @@ -532,7 +532,7 @@ def _deobfuscate_filenames(self, response: dict, file_map: dict) -> dict: deobfuscated[key] = value return deobfuscated - def scan_file_with_options( + def scan_file_with_options( # noqa: PLR0913 self, file: str, deps_file: str = None, @@ -603,7 +603,7 @@ def scan_file(self, file: str) -> bool: success = False return success - def scan_files(self, files: []) -> bool: + def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 """ Scan the specified list of files, producing fingerprints, send to the SCANOSS API and return results Please note that by providing an explicit list you bypass any exclusions that may be defined on the scanner @@ -657,7 +657,7 @@ def scan_files(self, files: []) -> bool: file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue + # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue # noqa: E501 if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -666,7 +666,7 @@ def scan_files(self, files: []) -> bool: scan_block += wfp scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -755,7 +755,7 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: success = False return success - def scan_wfp_file(self, file: str = None) -> bool: + def scan_wfp_file(self, file: str = None) -> bool: # noqa: PLR0912, PLR0915 """ Scan the contents of the specified WFP file (in the current process) :param file: Scan the contents of the specified WFP file (in the current process) diff --git a/src/scanoss/scanners/container_scanner.py b/src/scanoss/scanners/container_scanner.py index 43aec688..913d8804 100644 --- a/src/scanoss/scanners/container_scanner.py +++ b/src/scanoss/scanners/container_scanner.py @@ -436,10 +436,12 @@ def _format_cyclonedx_output(self) -> str: scan_results = {} for f in self.scanner.decorated_scan_results['files']: scan_results[f['file']] = [f] - if not cdx.produce_from_json(scan_results, self.output_file): + success, cdx_output = cdx.produce_from_json(scan_results) + if not success: error_msg = 'Failed to produce CycloneDX output' self.base.print_stderr(error_msg) - raise ValueError(error_msg) + return None + return json.dumps(cdx_output, indent=2) def _format_spdxlite_output(self) -> str: """ diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 092003d0..8f6615cc 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -196,14 +196,16 @@ def _format_cyclonedx_output(self) -> str: decorated_scan_results = self.scanner.client.get_dependencies(get_dependencies_json_request) - cdx = CycloneDx(self.base.debug, self.output_file) + cdx = CycloneDx(self.base.debug) scan_results = {} for f in decorated_scan_results['files']: scan_results[f['file']] = [f] - if not cdx.produce_from_json(scan_results, self.output_file): + success, cdx_output = cdx.produce_from_json(scan_results) + if not success: error_msg = 'ERROR: Failed to produce CycloneDX output' self.base.print_stderr(error_msg) - raise ValueError(error_msg) + return None + return json.dumps(cdx_output, indent=2) except Exception as e: self.base.print_stderr(f'ERROR: Failed to get license information: {e}') return None From be91ae6f16f4e01050e759b8854dfa4e996a8ee6 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 9 Jul 2025 10:16:02 +0200 Subject: [PATCH 367/489] [SP-2870] chore: lint errors --- src/scanoss/cyclonedx.py | 4 ++-- src/scanoss/scanners/scanner_hfh.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 4bf326b4..bc6ac661 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -48,7 +48,7 @@ def __init__(self, debug: bool = False, output_file: str = None): self.debug = debug self._spdx = SpdxLite(debug=debug) - def parse(self, data: json): + def parse(self, data: json): # noqa: PLR0912, PLR0915 """ Parse the given input (raw/plain) JSON string and return CycloneDX summary :param data: json - JSON object @@ -57,7 +57,7 @@ def parse(self, data: json): if not data: self.print_stderr('ERROR: No JSON data provided to parse.') return None, None - self.print_debug(f'Processing raw results into CycloneDX format...') + self.print_debug('Processing raw results into CycloneDX format...') cdx = {} vdx = {} for f in data: diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 8f6615cc..b6373c08 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -162,7 +162,7 @@ def _format_plain_output(self) -> str: else str(self.scanner.scan_results) ) - def _format_cyclonedx_output(self) -> str: + def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 if not self.scanner.scan_results: return '' try: From 41e1d16c5e5a491580c05c061289c1f51b246e86 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 10 Jul 2025 16:03:33 +0200 Subject: [PATCH 368/489] feat: add vulnerabilities into cdx output when using folder hashing command --- CHANGELOG.md | 4 ++ src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 69 +++++++++++++++++++++++++++++ src/scanoss/scanners/scanner_hfh.py | 9 ++++ src/scanoss/scanossgrpc.py | 2 +- 5 files changed, 84 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23578873..c83c5708 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.28.0] - 2025-07-10 +### Added +- Add vulnerabilities response to `folder-scan` CycloneDX output + ## [1.27.1] - 2025-07-09 ### Fixed - Fixed when running `folder-scan` with `--format cyclonedx` the output was not writing to file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index cc543c19..847e76af 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.27.1' +__version__ = '1.28.0' diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index bc6ac661..f952deb7 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -287,6 +287,75 @@ def produce_from_str(self, json_str: str, output_file: str = None) -> bool: return False return self.produce_from_json(data, output_file) + def append_vulnerabilities(self, cdx_dict: dict, vulnerabilities_data: dict, purl: str) -> dict: + """ + Append vulnerabilities to an existing CycloneDX dictionary + + Args: + cdx_dict (dict): The existing CycloneDX dictionary + vulnerabilities_data (dict): The vulnerabilities data from get_vulnerabilities_json + purl (str): The PURL of the component these vulnerabilities affect + + Returns: + dict: The updated CycloneDX dictionary with vulnerabilities appended + """ + if not cdx_dict or not vulnerabilities_data: + return cdx_dict + + if 'vulnerabilities' not in cdx_dict: + cdx_dict['vulnerabilities'] = [] + + # Extract vulnerabilities from the response + vulns_list = vulnerabilities_data.get('purls', []) + if vulns_list and len(vulns_list) > 0: + vuln_items = vulns_list[0].get('vulnerabilities', []) + + for vuln in vuln_items: + vuln_id = vuln.get('ID', '') + if vuln_id == '': + vuln_id = vuln.get('id', '') + if not vuln_id or vuln_id == '': # Skip empty ids + continue + + vuln_cve = vuln.get('CVE', '') + if vuln_cve == '': + vuln_cve = vuln.get('cve', '') + + # Skip CPE entries, use CVE if available + if vuln_id.upper().startswith('CPE:') and vuln_cve != '': + vuln_id = vuln_cve + elif vuln_id.upper().startswith('CPE:'): + continue + + # Check if vulnerability already exists + existing_vuln = None + for existing in cdx_dict['vulnerabilities']: + if existing.get('id') == vuln_id: + existing_vuln = existing + break + + if existing_vuln: + # Add this PURL to the affects list if not already present + if not any(ref.get('ref') == purl for ref in existing_vuln.get('affects', [])): + existing_vuln['affects'].append({'ref': purl}) + else: + # Create new vulnerability entry + vuln_source = vuln.get('source', '').lower() + vd = { + 'id': vuln_id, + 'source': { + 'name': 'NVD' if vuln_source == 'nvd' else 'GitHub Advisories', + 'url': f'https://nvd.nist.gov/vuln/detail/{vuln_cve}' + if vuln_source == 'nvd' + else f'https://github.com/advisories/{vuln_id}' + }, + 'ratings': [{'severity': self._sev_lookup(vuln.get('severity', 'unknown').lower())}], + 'affects': [{'ref': purl}] + } + cdx_dict['vulnerabilities'].append(vd) + + return cdx_dict + @staticmethod def _sev_lookup(value: str): """ diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index b6373c08..9f4df38c 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -193,8 +193,13 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 } ] } + + get_vulnerabilities_json_request = { + 'purls': [{'purl': purl, 'requirement': best_match_version['version']}], + } decorated_scan_results = self.scanner.client.get_dependencies(get_dependencies_json_request) + vulnerabilities = self.scanner.client.get_vulnerabilities_json(get_vulnerabilities_json_request) cdx = CycloneDx(self.base.debug) scan_results = {} @@ -205,6 +210,10 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 error_msg = 'ERROR: Failed to produce CycloneDX output' self.base.print_stderr(error_msg) return None + + if vulnerabilities: + cdx_output = cdx.append_vulnerabilities(cdx_output, vulnerabilities, purl) + return json.dumps(cdx_output, indent=2) except Exception as e: self.base.print_stderr(f'ERROR: Failed to get license information: {e}') diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 189f4c10..4c11f715 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -326,7 +326,7 @@ def get_vulnerabilities_json(self, purls: dict) -> dict: request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending crypto data for decoration (rqId: {request_id})...') + self.print_debug(f'Sending vulnerability data for decoration (rqId: {request_id})...') resp = self.vuln_stub.GetVulnerabilities(request, metadata=metadata, timeout=self.timeout) except Exception as e: self.print_stderr( From ec19a2eedd199eafb34a76df355e87843a0b8af2 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 10 Jul 2025 16:12:37 +0200 Subject: [PATCH 369/489] feat: refactor method --- src/scanoss/cyclonedx.py | 102 ++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index f952deb7..030dd3e5 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -287,6 +287,37 @@ def produce_from_str(self, json_str: str, output_file: str = None) -> bool: return False return self.produce_from_json(data, output_file) + def _normalize_vulnerability_id(self, vuln: dict) -> tuple[str, str]: + """ + Normalize vulnerability ID and CVE from different possible field names. + Returns tuple of (vuln_id, vuln_cve). + """ + vuln_id = vuln.get('ID', '') or vuln.get('id', '') + vuln_cve = vuln.get('CVE', '') or vuln.get('cve', '') + + # Skip CPE entries, use CVE if available + if vuln_id.upper().startswith('CPE:') and vuln_cve: + vuln_id = vuln_cve + + return vuln_id, vuln_cve + + def _create_vulnerability_entry(self, vuln_id: str, vuln: dict, vuln_cve: str, purl: str) -> dict: + """ + Create a new vulnerability entry for CycloneDX format. + """ + vuln_source = vuln.get('source', '').lower() + return { + 'id': vuln_id, + 'source': { + 'name': 'NVD' if vuln_source == 'nvd' else 'GitHub Advisories', + 'url': f'https://nvd.nist.gov/vuln/detail/{vuln_cve}' + if vuln_source == 'nvd' + else f'https://github.com/advisories/{vuln_id}' + }, + 'ratings': [{'severity': self._sev_lookup(vuln.get('severity', 'unknown').lower())}], + 'affects': [{'ref': purl}] + } + def append_vulnerabilities(self, cdx_dict: dict, vulnerabilities_data: dict, purl: str) -> dict: """ Append vulnerabilities to an existing CycloneDX dictionary @@ -307,52 +338,33 @@ def append_vulnerabilities(self, cdx_dict: dict, vulnerabilities_data: dict, pur # Extract vulnerabilities from the response vulns_list = vulnerabilities_data.get('purls', []) - if vulns_list and len(vulns_list) > 0: - vuln_items = vulns_list[0].get('vulnerabilities', []) + if not vulns_list: + return cdx_dict - for vuln in vuln_items: - vuln_id = vuln.get('ID', '') - if vuln_id == '': - vuln_id = vuln.get('id', '') - if not vuln_id or vuln_id == '': # Skip empty ids - continue - - vuln_cve = vuln.get('CVE', '') - if vuln_cve == '': - vuln_cve = vuln.get('cve', '') - - # Skip CPE entries, use CVE if available - if vuln_id.upper().startswith('CPE:') and vuln_cve != '': - vuln_id = vuln_cve - elif vuln_id.upper().startswith('CPE:'): - continue - - # Check if vulnerability already exists - existing_vuln = None - for existing in cdx_dict['vulnerabilities']: - if existing.get('id') == vuln_id: - existing_vuln = existing - break - - if existing_vuln: - # Add this PURL to the affects list if not already present - if not any(ref.get('ref') == purl for ref in existing_vuln.get('affects', [])): - existing_vuln['affects'].append({'ref': purl}) - else: - # Create new vulnerability entry - vuln_source = vuln.get('source', '').lower() - vd = { - 'id': vuln_id, - 'source': { - 'name': 'NVD' if vuln_source == 'nvd' else 'GitHub Advisories', - 'url': f'https://nvd.nist.gov/vuln/detail/{vuln_cve}' - if vuln_source == 'nvd' - else f'https://github.com/advisories/{vuln_id}' - }, - 'ratings': [{'severity': self._sev_lookup(vuln.get('severity', 'unknown').lower())}], - 'affects': [{'ref': purl}] - } - cdx_dict['vulnerabilities'].append(vd) + vuln_items = vulns_list[0].get('vulnerabilities', []) + + for vuln in vuln_items: + vuln_id, vuln_cve = self._normalize_vulnerability_id(vuln) + + # Skip empty IDs or CPE-only entries + if not vuln_id or vuln_id.upper().startswith('CPE:'): + continue + + # Check if vulnerability already exists + existing_vuln = next( + (v for v in cdx_dict['vulnerabilities'] if v.get('id') == vuln_id), + None + ) + + if existing_vuln: + # Add this PURL to the affects list if not already present + if not any(ref.get('ref') == purl for ref in existing_vuln.get('affects', [])): + existing_vuln['affects'].append({'ref': purl}) + else: + # Create new vulnerability entry + cdx_dict['vulnerabilities'].append( + self._create_vulnerability_entry(vuln_id, vuln, vuln_cve, purl) + ) return cdx_dict From 9f3bbe56d4b962348c8a860e35b8cff8e63c377a Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 10 Jul 2025 16:20:19 +0200 Subject: [PATCH 370/489] fix(crypto): dict object error in --input processing --- src/scanoss/cryptography.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index 7fcc5989..da0ed033 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -62,7 +62,7 @@ def __post_init__(self): for purl in purls: purls_with_requirement.append(f'{purl["purl"]}@{purl["requirement"]}') else: - purls_with_requirement = purls + purls_with_requirement = [purl["purl"] for purl in purls] self.purl = purls_with_requirement From 116e59f993ea41c6b0e2ca1301b0647921618047 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 10 Jul 2025 17:18:30 +0200 Subject: [PATCH 371/489] fix: purls parsing error --- src/scanoss/cryptography.py | 38 +++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index da0ed033..9f367ba3 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -53,16 +53,16 @@ def __post_init__(self): raise ScanossCryptographyError('The supplied input file is not in the correct PurlRequest format.') purls = input_file_validation.data['purls'] purls_with_requirement = [] - if self.with_range: - if any('requirement' not in p for p in purls): - raise ScanossCryptographyError( - f'One or more PURLs in "{self.input_file}" are missing the "requirement" field.' - ) + if self.with_range and any('requirement' not in p for p in purls): + raise ScanossCryptographyError( + f'One or more PURLs in "{self.input_file}" are missing the "requirement" field.' + ) + + for purl in purls: + if 'requirement' in purl: + purls_with_requirement.append(f'{purl["purl"]}@{purl["requirement"]}') else: - for purl in purls: - purls_with_requirement.append(f'{purl["purl"]}@{purl["requirement"]}') - else: - purls_with_requirement = [purl["purl"] for purl in purls] + purls_with_requirement.append(purl['purl']) self.purl = purls_with_requirement @@ -198,13 +198,27 @@ def _build_purls_request( return { 'purls': [ { - 'purl': p, - 'requirement': self._extract_version_from_purl(p), + 'purl': self._remove_version_from_purl(purl), + 'requirement': self._extract_version_from_purl(purl), } - for p in self.config.purl + for purl in self.config.purl ] } + def _remove_version_from_purl(self, purl: str) -> str: + """ + Remove version from purl + + Args: + purl (str): The purl string to remove the version from + + Returns: + str: The purl string without the version + """ + if '@' not in purl: + return purl + return purl.split('@')[0] + def _extract_version_from_purl(self, purl: str) -> str: """ Extract version from purl From e34a56efdb6f45245ab572a4c1e942a279b219aa Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 10 Jul 2025 17:26:33 +0200 Subject: [PATCH 372/489] chore: upgrade version --- CHANGELOG.md | 6 ++++++ src/scanoss/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c83c5708..00dada9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.28.1] - 2025-07-10 +### Added +- Fix purls parsing on `crypto` subcommand + ## [1.28.0] - 2025-07-10 ### Added - Add vulnerabilities response to `folder-scan` CycloneDX output @@ -587,3 +591,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.26.3]: https://github.com/scanoss/scanoss.py/compare/v1.26.2...v1.26.3 [1.27.0]: https://github.com/scanoss/scanoss.py/compare/v1.26.3...v1.27.0 [1.27.1]: https://github.com/scanoss/scanoss.py/compare/v1.27.0...v1.27.1 +[1.28.0]: https://github.com/scanoss/scanoss.py/compare/v1.27.1...v1.28.0 +[1.28.1]: https://github.com/scanoss/scanoss.py/compare/v1.28.0...v1.28.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 847e76af..7cb5f839 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.28.0' +__version__ = '1.28.1' From 4b08c5db324e61a1afdae7cae64940703e9dc6df Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 14 Jul 2025 13:05:16 +0200 Subject: [PATCH 373/489] [SP-2912] fix: cdx format error when license id is none --- src/scanoss/cyclonedx.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 030dd3e5..afb9f2e1 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -219,6 +219,8 @@ def produce_from_json(self, data: json, output_file: str = None) -> tuple[bool, lic_set = set() for lic in licenses: # Get a unique set of licenses lc_id = lic.get('id') + if not lc_id: + continue spdx_id = self._spdx.get_spdx_license_id(lc_id) lic_set.add(spdx_id if spdx_id else lc_id) for lc_id in lic_set: # Store licenses for later inclusion From 60071c309396ccc82f18357e9f05b18f1bdf16de Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 14 Jul 2025 14:36:58 +0200 Subject: [PATCH 374/489] [SP-2912] chore: bump version --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00dada9f..24d56a4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.28.2] - 2025-07-14 +### Fixed +- Fix CycloneDX format when license id is None + ## [1.28.1] - 2025-07-10 ### Added - Fix purls parsing on `crypto` subcommand @@ -593,3 +597,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.27.1]: https://github.com/scanoss/scanoss.py/compare/v1.27.0...v1.27.1 [1.28.0]: https://github.com/scanoss/scanoss.py/compare/v1.27.1...v1.28.0 [1.28.1]: https://github.com/scanoss/scanoss.py/compare/v1.28.0...v1.28.1 +[1.28.2]: https://github.com/scanoss/scanoss.py/compare/v1.28.1...v1.28.2 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 7cb5f839..8530ab3a 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.28.1' +__version__ = '1.28.2' From fae200e34b5f8f4a24618d0599d2804b1680c92c Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 14 Jul 2025 17:05:21 +0200 Subject: [PATCH 375/489] feat: implement exclude settings co-authored-by: Agustin Groh --- src/scanoss/cli.py | 5 ++--- src/scanoss/scanoss_settings.py | 36 ++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index fc618eb4..3ac1f263 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1071,9 +1071,8 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 'blacklist' ) else: - scan_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new').set_scan_type( - 'identify' - ) + scan_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new') + except ScanossSettingsError as e: print_stderr(f'Error: {e}') sys.exit(1) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 1770697d..ff9b9292 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -172,7 +172,7 @@ def _is_valid_sbom_file(self): def _get_bom(self): """ - Get the Billing of Materials from the settings file + Get the Bill of Materials from the settings file Returns: dict: If using scanoss.json list: If using SBOM.json @@ -196,6 +196,17 @@ def get_bom_include(self) -> List[BomEntry]: return self._get_bom() return self._get_bom().get('include', []) + + def get_bom_exclude(self) -> List[BomEntry]: + """ + Get the list of components to exclude from the scan + Returns: + list: List of components to exclude from the scan + """ + if self.settings_file_type == 'legacy': + return self._get_bom() + return self._get_bom().get('exclude', []) + def get_bom_remove(self) -> List[BomEntry]: """ Get the list of components to remove from the scan @@ -225,8 +236,8 @@ def get_sbom(self): if not self.data: return None return { - 'scan_type': self.scan_type, 'assets': json.dumps(self._get_sbom_assets()), + 'scan_type': self.scan_type, } def _get_sbom_assets(self): @@ -235,7 +246,18 @@ def _get_sbom_assets(self): Returns: List: List of SBOM assets """ - if self.scan_type == 'identify': + + if self.settings_file_type == 'new': + if len(self.get_bom_include()): + self.scan_type = 'identify' + include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include())) + return {"components": include_bom_entries} + elif len(self.get_bom_exclude()): + self.scan_type = 'blacklist' + exclude_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_exclude())) + return {"components": exclude_bom_entries} + + if self.settings_file_type == 'legacy' and self.scan_type == 'identify': # sbom-identify.json include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include())) replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace())) self.print_debug( @@ -244,6 +266,14 @@ def _get_sbom_assets(self): f'From Replace list: {[entry["purl"] for entry in replace_bom_entries]} \n' ) return include_bom_entries + replace_bom_entries + + if self.settings_file_type == 'legacy' and self.scan_type == 'blacklist': # sbom-identify.json + exclude_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_exclude())) + self.print_debug( + f"Scan type set to 'blacklist'. Adding {len(exclude_bom_entries)} components as context to the scan. \n" # noqa: E501 + f'From Exclude list: {[entry["purl"] for entry in exclude_bom_entries]} \n') + return exclude_bom_entries + return self.normalize_bom_entries(self.get_bom_remove()) @staticmethod From 589259c82b2e334bd06c88266c5e6dfffe49f758 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Mon, 14 Jul 2025 13:24:36 -0300 Subject: [PATCH 376/489] chore:Upgrades app version to v1.28.3 --- CHANGELOG.md | 6 ++++++ src/scanoss/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24d56a4a..53dd7ce0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.28.3] - 2025-07-14 +### Fixed +- Fixed scanoss.json ingestion +### Added +- Added support for exclude parameter from scanoss.json file during scanning + ## [1.28.2] - 2025-07-14 ### Fixed - Fix CycloneDX format when license id is None diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 8530ab3a..cdd9d0c5 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.28.2' +__version__ = '1.28.3' From a406ab2a4439ea3b8d7426c9859bcf4887667622 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 15 Jul 2025 09:18:50 +0200 Subject: [PATCH 377/489] chore: update minimum python version to 3.9 --- .github/workflows/container-local-test.yml | 2 +- .github/workflows/container-publish-ghcr.yml | 5 +++-- .github/workflows/lint.yml | 2 +- .github/workflows/python-local-test.yml | 2 +- .github/workflows/python-publish-pypi.yml | 11 ++++++----- .github/workflows/python-publish-testpypi.yml | 9 +++++---- .github/workflows/version-tag.yml | 4 ++-- CHANGELOG.md | 6 ++++++ PACKAGE.md | 2 +- README.md | 2 +- docs/source/index.rst | 2 +- pyproject.toml | 4 ++-- setup.cfg | 2 +- src/scanoss/__init__.py | 2 +- 14 files changed, 32 insertions(+), 23 deletions(-) diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 9eeead8c..6021e2c0 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: '3.9.x' - name: Install Dependencies run: | diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 5edb8585..14125a6e 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: push: tags: - - 'v*.*.*' + - "v*.*.*" env: REGISTRY: ghcr.io @@ -30,7 +30,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: "3.9.x" - name: Install Dependencies run: | @@ -159,3 +159,4 @@ jobs: # env: # TAGS: ${{ steps.meta.outputs.tags }} # COSIGN_EXPERIMENTAL: true + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3925cc90..842d6f76 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.9" - name: Install Dependencies run: | diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 12b001b3..24f36026 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.10.x" + python-version: "3.9.x" - name: Install Dependencies run: | diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 5cc934e5..793692c3 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: push: tags: - - 'v*.*.*' + - "v*.*.*" jobs: deploy: @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: "3.9.x" - name: Install dependencies run: | @@ -49,7 +49,7 @@ jobs: - name: Publish Package - ${{ github.ref_name }} uses: pypa/gh-action-pypi-publish@release/v1 with: -# skip-existing: true + # skip-existing: true user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} @@ -62,7 +62,7 @@ jobs: test: if: success() - needs: [ deploy ] + needs: [deploy] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -70,7 +70,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: "3.9.x" - name: Install Remote Package uses: nick-fields/retry@v3 @@ -113,3 +113,4 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index 0c105c7f..e0ecfa1c 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -1,7 +1,7 @@ name: Publish Python Package - TestPyPI # This workflow will upload a TestPyPI Python Package using Twine on demand (dispatch) -on: [ workflow_dispatch ] +on: [workflow_dispatch] permissions: contents: read @@ -15,7 +15,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: "3.9.x" - name: Install Dependencies run: | @@ -56,7 +56,7 @@ jobs: test: if: success() - needs: [ deploy ] + needs: [deploy] runs-on: ubuntu-latest steps: @@ -65,7 +65,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: "3.9.x" - name: Install Remote Package run: | @@ -101,3 +101,4 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + diff --git a/.github/workflows/version-tag.yml b/.github/workflows/version-tag.yml index 895068a7..0d333050 100644 --- a/.github/workflows/version-tag.yml +++ b/.github/workflows/version-tag.yml @@ -18,12 +18,12 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: '0' + fetch-depth: "0" token: ${{ secrets.SC_GH_TAG_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10.x' + python-version: "3.9.x" - name: Determine Tag id: taggerVersion run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 53dd7ce0..f71ee07a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.29.0] - 2025-07-15 +### Changed +- Updated minimum Python version to 3.9 + ## [1.28.3] - 2025-07-14 ### Fixed - Fixed scanoss.json ingestion @@ -604,3 +608,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.28.0]: https://github.com/scanoss/scanoss.py/compare/v1.27.1...v1.28.0 [1.28.1]: https://github.com/scanoss/scanoss.py/compare/v1.28.0...v1.28.1 [1.28.2]: https://github.com/scanoss/scanoss.py/compare/v1.28.1...v1.28.2 +[1.29.0]: https://github.com/scanoss/scanoss.py/compare/v1.28.2...v1.29.0 + diff --git a/PACKAGE.md b/PACKAGE.md index 7e4f8b26..620f2984 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -138,7 +138,7 @@ if __name__ == "__main__": ``` ## Requirements -Python 3.7 or higher. +Python 3.9 or higher. ## Source code The source for this package can be found [here](https://github.com/scanoss/scanoss.py). diff --git a/README.md b/README.md index a95f6277..6ca949d1 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ To leverage the CLI from within a container, please look at [GHCR.md](GHCR.md). Before starting with development of this project, please read our [CONTRIBUTING](CONTRIBUTING.md) and [CODE OF CONDUCT](CODE_OF_CONDUCT.md). ### Requirements -Python 3.7 or higher. +Python 3.9 or higher. The dependencies can be found in the [requirements.txt](requirements.txt) and [requirements-dev.txt](requirements-dev.txt) files. diff --git a/docs/source/index.rst b/docs/source/index.rst index eb6a98f8..6bd8667c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -35,7 +35,7 @@ To install (from `pypi.org `_), run: ``pip3 i Requirements ------------ -Python 3.7 or higher. +Python 3.9 or higher. The dependencies can be found in the `requirements.txt `_ and `requirements-dev.txt `_ files. diff --git a/pyproject.toml b/pyproject.toml index d740f7c1..ccca9080 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,8 @@ build-backend = "setuptools.build_meta" # Enable pycodestyle (E), pyflakes (F), isort (I), pylint (PL) select = ["E", "F", "I", "PL"] line-length = 120 -# Assume Python 3.7+ -target-version = "py37" +# Assume Python 3.9+ +target-version = "py39" exclude = [ "tests/*", "test_*.py", diff --git a/setup.cfg b/setup.cfg index 5880873e..a1772504 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ packages = find_namespace: package_dir = = src include_package_data = True -python_requires = >=3.7 +python_requires = >=3.9 install_requires = requests crc32c>=2.2 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index cdd9d0c5..327a9cef 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.28.3' +__version__ = '1.29.0' From cec64a71e3145a56ce0f40ecd60ef261d445b33f Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 15 Jul 2025 09:27:28 +0200 Subject: [PATCH 378/489] fix: use dict instead of json type --- src/scanoss/cyclonedx.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index afb9f2e1..5ef729e8 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -48,10 +48,10 @@ def __init__(self, debug: bool = False, output_file: str = None): self.debug = debug self._spdx = SpdxLite(debug=debug) - def parse(self, data: json): # noqa: PLR0912, PLR0915 + def parse(self, data: dict): # noqa: PLR0912, PLR0915 """ Parse the given input (raw/plain) JSON string and return CycloneDX summary - :param data: json - JSON object + :param data: dict - JSON object :return: CycloneDX dictionary, and vulnerability dictionary """ if not data: @@ -170,12 +170,12 @@ def produce_from_file(self, json_file: str, output_file: str = None) -> bool: success = self.produce_from_str(f.read(), output_file) return success - def produce_from_json(self, data: json, output_file: str = None) -> tuple[bool, json]: # noqa: PLR0912 + def produce_from_json(self, data: dict, output_file: str = None) -> tuple[bool, dict]: # noqa: PLR0912 """ Produce the CycloneDX output from the raw scan results input data Args: - data (json): JSON object + data (dict): JSON object output_file (str, optional): Output file (optional). Defaults to None. Returns: From ea42cd6942473fc711ed2029a95539a19d9e555c Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 22 Jul 2025 07:32:10 +0200 Subject: [PATCH 379/489] [SP-2879] feat: add export dt sub-command, add cyclonedx input file validation --- CHANGELOG.md | 7 + requirements.txt | 4 +- setup.cfg | 1 + src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 120 +++++++++++--- src/scanoss/cyclonedx.py | 65 +++++--- src/scanoss/export/__init__.py | 27 +++ src/scanoss/export/dependency_track.py | 221 +++++++++++++++++++++++++ 8 files changed, 400 insertions(+), 47 deletions(-) create mode 100644 src/scanoss/export/__init__.py create mode 100644 src/scanoss/export/dependency_track.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f71ee07a..c2b1101c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.30.0] - 2025-07-22 +### Added +- Add `export dt` subcommand to export SBOM files to Dependency Track +- Add CycloneDX file validation + ## [1.29.0] - 2025-07-15 ### Changed - Updated minimum Python version to 3.9 @@ -609,4 +614,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.28.1]: https://github.com/scanoss/scanoss.py/compare/v1.28.0...v1.28.1 [1.28.2]: https://github.com/scanoss/scanoss.py/compare/v1.28.1...v1.28.2 [1.29.0]: https://github.com/scanoss/scanoss.py/compare/v1.28.2...v1.29.0 +[1.30.0]: https://github.com/scanoss/scanoss.py/compare/v1.29.0...v1.30.0 + diff --git a/requirements.txt b/requirements.txt index 70db9fa6..4add4629 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,6 @@ importlib_resources packageurl-python pathspec jsonschema -crc \ No newline at end of file +crc + +cyclonedx-python-lib[validation] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a1772504..36e0d74b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,7 @@ install_requires = pathspec jsonschema crc + cyclonedx-python-lib[validation] [options.extras_require] diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 327a9cef..eaa95be6 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.29.0' +__version__ = '1.30.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 3ac1f263..a408d80f 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -25,6 +25,7 @@ import argparse import os import sys +import traceback from dataclasses import asdict from pathlib import Path from typing import List @@ -32,6 +33,10 @@ import pypac from scanoss.cryptography import Cryptography, create_cryptography_config_from_args +from scanoss.export.dependency_track import ( + DependencyTrackExporter, + create_dependency_track_exporter_config_from_args, +) from scanoss.inspection.component_summary import ComponentSummary from scanoss.inspection.license_summary import LicenseSummary from scanoss.scanners.container_scanner import ( @@ -553,13 +558,17 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ####### INSPECT: License Summary ###### # Inspect Sub-command: inspect license summary p_license_summary = p_inspect_sub.add_parser( - 'license-summary', aliases=['lic-summary', 'licsum'], description='Get license summary', - help='Get detected license summary from scan results' + 'license-summary', + aliases=['lic-summary', 'licsum'], + description='Get license summary', + help='Get detected license summary from scan results', ) p_component_summary = p_inspect_sub.add_parser( - 'component-summary', aliases=['comp-summary', 'compsum'], description='Get component summary', - help='Get detected component summary from scan results' + 'component-summary', + aliases=['comp-summary', 'compsum'], + description='Get component summary', + help='Get detected component summary from scan results', ) ####### INSPECT: Undeclared components ###### @@ -605,6 +614,36 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ########################################### END INSPECT SUBCOMMAND ########################################### + # Sub-command: export + p_export = subparsers.add_parser( + 'export', + aliases=['exp'], + description=f'Export SBOM files to external platforms: {__version__}', + help='Export SBOM files to external platforms', + ) + + export_sub = p_export.add_subparsers( + title='Export Commands', + dest='subparsercmd', + description='export sub-commands', + help='export sub-commands', + ) + + # Export Sub-command: export dt (Dependency Track) + e_dt = export_sub.add_parser( + 'dt', + aliases=['dependency-track'], + description='Export SBOM to Dependency Track', + help='Upload SBOM files to Dependency Track', + ) + e_dt.add_argument('-i', '--input', type=str, required=True, help='Input SBOM file (CycloneDX JSON format)') + e_dt.add_argument('--dt-url', type=str, required=True, help='Dependency Track base URL') + e_dt.add_argument('--dt-apikey', type=str, required=True, help='Dependency Track API key') + e_dt.add_argument('--dt-projectid', type=str, help='Dependency Track project UUID') + e_dt.add_argument('--dt-projectname', type=str, help='Dependency Track project name') + e_dt.add_argument('--dt-projectversion', type=str, help='Dependency Track project version') + e_dt.set_defaults(func=export_dt) + # Sub-command: folder-scan p_folder_scan = subparsers.add_parser( 'folder-scan', @@ -858,6 +897,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range, + e_dt, ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') @@ -871,7 +911,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 parser.print_help() # No sub command subcommand, print general help sys.exit(1) elif ( - args.subparser in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins', 'crypto', 'cr') + args.subparser + in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins', 'crypto', 'cr', 'export', 'exp') ) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed sys.exit(1) @@ -1304,6 +1345,7 @@ def convert(parser, args): if not success: sys.exit(1) + ################################ INSPECT handlers ################################ def inspect_copyleft(parser, args): """ @@ -1381,16 +1423,17 @@ def inspect_undeclared(parser, args): status, _ = i_undeclared.run() sys.exit(status) + def inspect_license_summary(parser, args): """ - Run the "inspect" sub-command - Parameters - ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments - """ + Run the "inspect" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ if args.input is None: print_stderr('Please specify an input file to inspect') parser.parse_args([args.subparser, args.subparsercmd, '-h']) @@ -1412,16 +1455,17 @@ def inspect_license_summary(parser, args): ) i_license_summary.run() + def inspect_component_summary(parser, args): """ - Run the "inspect" sub-command - Parameters - ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments - """ + Run the "inspect" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ if args.input is None: print_stderr('Please specify an input file to inspect') parser.parse_args([args.subparser, args.subparsercmd, '-h']) @@ -1440,8 +1484,42 @@ def inspect_component_summary(parser, args): ) i_component_summary.run() + ################################ End inspect handlers ################################ + +def export_dt(parser, args): + """ + Run the "export dt" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + + try: + config = create_dependency_track_exporter_config_from_args(args) + dt_exporter = DependencyTrackExporter( + config=config, + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + ) + + success = dt_exporter.upload_sbom(args.input) + + if not success: + sys.exit(1) + + except Exception as e: + print_stderr(f'ERROR: {e}') + if args.debug: + traceback.print_exc() + sys.exit(1) + + def utils_certloc(*_): """ Run the "utils certloc" sub-command diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 5ef729e8..d6ad389a 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -28,6 +28,9 @@ import sys import uuid +from cyclonedx.schema import SchemaVersion +from cyclonedx.validation.json import JsonValidator + from . import __version__ from .scanossbase import ScanossBase from .spdxlite import SpdxLite @@ -296,13 +299,13 @@ def _normalize_vulnerability_id(self, vuln: dict) -> tuple[str, str]: """ vuln_id = vuln.get('ID', '') or vuln.get('id', '') vuln_cve = vuln.get('CVE', '') or vuln.get('cve', '') - + # Skip CPE entries, use CVE if available if vuln_id.upper().startswith('CPE:') and vuln_cve: vuln_id = vuln_cve - + return vuln_id, vuln_cve - + def _create_vulnerability_entry(self, vuln_id: str, vuln: dict, vuln_cve: str, purl: str) -> dict: """ Create a new vulnerability entry for CycloneDX format. @@ -313,61 +316,56 @@ def _create_vulnerability_entry(self, vuln_id: str, vuln: dict, vuln_cve: str, p 'source': { 'name': 'NVD' if vuln_source == 'nvd' else 'GitHub Advisories', 'url': f'https://nvd.nist.gov/vuln/detail/{vuln_cve}' - if vuln_source == 'nvd' - else f'https://github.com/advisories/{vuln_id}' + if vuln_source == 'nvd' + else f'https://github.com/advisories/{vuln_id}', }, 'ratings': [{'severity': self._sev_lookup(vuln.get('severity', 'unknown').lower())}], - 'affects': [{'ref': purl}] + 'affects': [{'ref': purl}], } - + def append_vulnerabilities(self, cdx_dict: dict, vulnerabilities_data: dict, purl: str) -> dict: """ Append vulnerabilities to an existing CycloneDX dictionary - + Args: cdx_dict (dict): The existing CycloneDX dictionary vulnerabilities_data (dict): The vulnerabilities data from get_vulnerabilities_json purl (str): The PURL of the component these vulnerabilities affect - + Returns: dict: The updated CycloneDX dictionary with vulnerabilities appended """ if not cdx_dict or not vulnerabilities_data: return cdx_dict - + if 'vulnerabilities' not in cdx_dict: cdx_dict['vulnerabilities'] = [] - + # Extract vulnerabilities from the response vulns_list = vulnerabilities_data.get('purls', []) if not vulns_list: return cdx_dict - + vuln_items = vulns_list[0].get('vulnerabilities', []) - + for vuln in vuln_items: vuln_id, vuln_cve = self._normalize_vulnerability_id(vuln) - + # Skip empty IDs or CPE-only entries if not vuln_id or vuln_id.upper().startswith('CPE:'): continue - + # Check if vulnerability already exists - existing_vuln = next( - (v for v in cdx_dict['vulnerabilities'] if v.get('id') == vuln_id), - None - ) - + existing_vuln = next((v for v in cdx_dict['vulnerabilities'] if v.get('id') == vuln_id), None) + if existing_vuln: # Add this PURL to the affects list if not already present if not any(ref.get('ref') == purl for ref in existing_vuln.get('affects', [])): existing_vuln['affects'].append({'ref': purl}) else: # Create new vulnerability entry - cdx_dict['vulnerabilities'].append( - self._create_vulnerability_entry(vuln_id, vuln, vuln_cve, purl) - ) - + cdx_dict['vulnerabilities'].append(self._create_vulnerability_entry(vuln_id, vuln, vuln_cve, purl)) + return cdx_dict @staticmethod @@ -388,6 +386,25 @@ def _sev_lookup(value: str): 'unknown': 'unknown', }.get(value, 'unknown') + def is_cyclonedx_json(self, json_string: str) -> bool: + """ + Validate if the given JSON string is a valid CycloneDX JSON string + Args: + json_string (str): JSON string to validate + Returns: + bool: True if the JSON string is valid, False otherwise + """ + try: + cdx_json_validator = JsonValidator(SchemaVersion.V1_6) + json_validation_errors = cdx_json_validator.validate_str(json_string) + if json_validation_errors: + self.print_stderr(f'ERROR: Problem parsing input JSON: {json_validation_errors}') + return False + return True + except Exception as e: + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + return False + # # End of CycloneDX Class diff --git a/src/scanoss/export/__init__.py b/src/scanoss/export/__init__.py new file mode 100644 index 00000000..0f8a5426 --- /dev/null +++ b/src/scanoss/export/__init__.py @@ -0,0 +1,27 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +from .dependency_track import DependencyTrackExporter + +__all__ = ['DependencyTrackExporter'] \ No newline at end of file diff --git a/src/scanoss/export/dependency_track.py b/src/scanoss/export/dependency_track.py new file mode 100644 index 00000000..acaeb623 --- /dev/null +++ b/src/scanoss/export/dependency_track.py @@ -0,0 +1,221 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import base64 +import json +import traceback +from dataclasses import dataclass +from typing import Optional + +import requests + +from scanoss.cyclonedx import CycloneDx + +from ..scanossbase import ScanossBase +from ..utils.file import validate_json_file + + +@dataclass +class DependencyTrackExporterConfig: + debug: bool = False + trace: bool = False + quiet: bool = False + dt_url: str = None + dt_apikey: str = None + dt_projectid: Optional[str] = None + dt_projectname: Optional[str] = None + dt_projectversion: Optional[str] = None + + +def create_dependency_track_exporter_config_from_args(args) -> DependencyTrackExporterConfig: + return DependencyTrackExporterConfig( + debug=getattr(args, 'debug', False), + trace=getattr(args, 'trace', False), + quiet=getattr(args, 'quiet', False), + dt_url=getattr(args, 'dt_url', None), + dt_apikey=getattr(args, 'dt_apikey', None), + dt_projectid=getattr(args, 'dt_projectid', None), + dt_projectname=getattr(args, 'dt_projectname', None), + dt_projectversion=getattr(args, 'dt_projectversion', None), + ) + + +class DependencyTrackExporter(ScanossBase): + """ + Class for exporting SBOM files to Dependency Track + """ + + def __init__( + self, + config: DependencyTrackExporterConfig, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + ): + """ + Initialize DependencyTrackExporter + + Args: + config: Configuration parameters for the dependency track exporter + debug: Enable debug output + trace: Enable trace output + quiet: Enable quiet mode + """ + super().__init__(debug=debug, trace=trace, quiet=quiet) + + self.dt_url = config.dt_url.rstrip('/') + self.dt_apikey = config.dt_apikey + self.dt_projectid = config.dt_projectid + self.dt_projectname = config.dt_projectname + self.dt_projectversion = config.dt_projectversion + + self._validate_config() + + def _validate_config(self): + """ + Validate that the configuration is valid. + """ + has_id = bool(self.dt_projectid) + has_name_version = bool(self.dt_projectname and self.dt_projectversion) + + if not (has_id or has_name_version): + raise ValueError('Either --dt-projectid OR (--dt-projectname and --dt-projectversion) must be provided') + + if has_id and has_name_version: + self.print_debug('Both DT project ID and name/version provided. Using project ID.') + + def _read_and_validate_sbom(self, input_file: str) -> dict: + """ + Read and validate the SBOM file + + Args: + input_file: Path to the SBOM file + + Returns: + Parsed SBOM content as dictionary + + Raises: + ValueError: If file doesn't exist or is invalid or not a valid CycloneDX SBOM + """ + result = validate_json_file(input_file) + if not result.is_valid: + raise ValueError(f'Invalid JSON file: {result.error}') + + cdx = CycloneDx(debug=self.debug) + if not cdx.is_cyclonedx_json(json.dumps(result.data)): + raise ValueError(f'Input file is not a valid CycloneDX SBOM: {input_file}') + + return result.data + + def _encode_sbom(self, sbom_content: dict) -> str: + """ + Encode SBOM content to base64 + + Args: + sbom_content: SBOM dictionary + + Returns: + Base64 encoded string + """ + json_str = json.dumps(sbom_content, separators=(',', ':')) + encoded = base64.b64encode(json_str.encode('utf-8')).decode('utf-8') + return encoded + + def _build_payload(self, encoded_sbom: str) -> dict: + """ + Build the API payload + + Args: + encoded_sbom: Base64 encoded SBOM + + Returns: + API payload dictionary + """ + if self.dt_projectid: + return {'project': self.dt_projectid, 'bom': encoded_sbom} + else: + return { + 'projectName': self.dt_projectname, + 'projectVersion': self.dt_projectversion, + 'autoCreate': True, + 'bom': encoded_sbom, + } + + def upload_sbom(self, input_file: str) -> bool: + """ + Upload SBOM file to Dependency Track + + Args: + input_file: Path to the SBOM file + + Returns: + True if successful, False otherwise + """ + try: + self.print_stderr(f'Reading SBOM file: {input_file}') + sbom_content = self._read_and_validate_sbom(input_file) + + self.print_debug('Encoding SBOM to base64') + encoded_sbom = self._encode_sbom(sbom_content) + + payload = self._build_payload(encoded_sbom) + + url = f'{self.dt_url}/api/v1/bom' + headers = {'Content-Type': 'application/json', 'X-Api-Key': self.dt_apikey} + + if self.trace: + self.print_trace(f'URL: {url}') + self.print_trace(f'Headers: {headers}') + self.print_trace(f'Payload keys: {list(payload.keys())}') + + self.print_msg('Uploading SBOM to Dependency Track...') + response = requests.put(url, json=payload, headers=headers) + + if response.status_code in [200, 201]: + self.print_stderr('SBOM uploaded successfully') + + try: + response_data = response.json() + if 'token' in response_data: + self.print_stderr(f'Upload token: {response_data["token"]}') + except json.JSONDecodeError: + pass + + return True + else: + self.print_stderr(f'Upload failed with status code: {response.status_code}') + self.print_stderr(f'Response: {response.text}') + return False + + except ValueError as e: + self.print_stderr(f'Validation error: {e}') + return False + except requests.exceptions.RequestException as e: + self.print_stderr(f'Request error: {e}') + return False + except Exception as e: + self.print_stderr(f'Unexpected error: {e}') + if self.debug: + traceback.print_exc() + return False From f36a7132050c0f49f2ab37a8afe4e815b07ad513 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Tue, 22 Jul 2025 12:28:56 +0200 Subject: [PATCH 380/489] [SP-2879] chore: update __init__.py --- src/scanoss/export/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/scanoss/export/__init__.py b/src/scanoss/export/__init__.py index 0f8a5426..1e95c46d 100644 --- a/src/scanoss/export/__init__.py +++ b/src/scanoss/export/__init__.py @@ -1,7 +1,7 @@ """ SPDX-License-Identifier: MIT - Copyright (c) 2024, SCANOSS + Copyright (c) 2025, SCANOSS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -21,7 +21,3 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - -from .dependency_track import DependencyTrackExporter - -__all__ = ['DependencyTrackExporter'] \ No newline at end of file From 5f9116203b7ea030b89ce1bcd8fc48898d9e5c60 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 24 Mar 2025 14:50:39 +0100 Subject: [PATCH 381/489] chore: add pre-commit config, add scanoss.json --- .github/workflows/scanoss.yml | 31 +++++++++++++++++++++++++++++++ .gitignore | 1 + .pre-commit-config.yaml | 5 +++++ scanoss.json | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+) create mode 100644 .github/workflows/scanoss.yml create mode 100644 scanoss.json diff --git a/.github/workflows/scanoss.yml b/.github/workflows/scanoss.yml new file mode 100644 index 00000000..d9dcb338 --- /dev/null +++ b/.github/workflows/scanoss.yml @@ -0,0 +1,31 @@ +name: SCANOSS + +on: + pull_request: + push: + branches: + - "*" + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + checks: write + actions: read + +jobs: + scanoss-code-scan: + name: SCANOSS Code Scan + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run SCANOSS Code Scan + id: scanoss-code-scan-step + uses: scanoss/code-scan-action@v1 + with: + policies: undeclared + api.url: https://api.scanoss.com/scan/direct + api.key: ${{ secrets.SC_API_KEY }} diff --git a/.gitignore b/.gitignore index edeb3710..fe84723a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ docs/build !docs/source/_static/*.json !scanoss-settings-schema.json .DS_Store +!scanoss.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bb2e62f0..da718915 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,3 +4,8 @@ repos: hooks: - id: ruff - id: ruff-format + - repo: https://github.com/scanoss/pre-commit-hooks + rev: v0.2.0 + hooks: + - id: scanoss-check-undeclared-code + diff --git a/scanoss.json b/scanoss.json new file mode 100644 index 00000000..37724c38 --- /dev/null +++ b/scanoss.json @@ -0,0 +1,33 @@ +{ + "settings": { + "skip": { + "patterns": {}, + "sizes": {} + } + }, + "bom": { + "include": [ + { + "purl": "pkg:github/scanoss/scanoss.py" + } + ], + "remove": [ + { + "path": "docs/make.bat", + "purl": "pkg:github/twilight-logic/ar488" + }, + { + "path": "src/protoc_gen_swagger/options/annotations_pb2_grpc.py", + "purl": "pkg:pypi/bauplan" + }, + { + "path": "src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py", + "purl": "pkg:pypi/bauplan" + }, + { + "path": "src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py", + "purl": "pkg:pypi/bauplan" + } + ] + } +} \ No newline at end of file From bb60a432836340b1cf3612b83f6b34b8451e2ee3 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Mon, 24 Mar 2025 14:52:14 +0100 Subject: [PATCH 382/489] chore: update readme --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 6ca949d1..f5d1bfb2 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,15 @@ To enable dependency scanning, an extra tool is required: scancode-toolkit pip3 install -r requirements-scancode.txt ``` +### Pre-commit Setup +This project uses pre-commit hooks to ensure code quality and consistency. To set up pre-commit, run: +```bash +pip3 install pre-commit +pre-commit install +``` + +This will install the pre-commit tool and set up the git hooks defined in the `.pre-commit-config.yaml` file to run automatically on each commit. + ### Devcontainer Setup To simplify the development environment setup, a devcontainer configuration is provided. This allows you to develop inside a containerized environment with all necessary dependencies pre-installed. From 752788e034148ed32fe3b115a44ad4817ec6af9d Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Fri, 8 Aug 2025 08:13:39 -0300 Subject: [PATCH 383/489] Feat/dep track inspection (#140) * chore:SP-2960 Refactor inspect module * feat:SP-2961 Implements dependency track inspect task * chore:SP-2967 Adds source(raw, dependency-track) to inspect subcommand * chore:SP-2969 Adds backward compatibility for the legacy inspect command * Fixed bug SP-2985 --------- Co-authored-by: Alex Egan Co-authored-by: eeisegn <44410969+eeisegn@users.noreply.github.com> --- CHANGELOG.md | 14 +- CLIENT_HELP.md | 18 +- Dockerfile | 6 +- docs/source/index.rst | 2 +- pyproject.toml | 5 +- src/__init__.py | 0 src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 761 +++++++++++++----- src/scanoss/export/dependency_track.py | 201 ++--- src/scanoss/file_filters.py | 6 +- .../dependency_track/project_violation.py | 443 ++++++++++ src/scanoss/inspection/policy_check.py | 77 +- .../inspection/{ => raw}/component_summary.py | 6 +- src/scanoss/inspection/{ => raw}/copyleft.py | 117 +-- .../inspection/{ => raw}/license_summary.py | 9 +- .../{inspect_base.py => raw/raw_base.py} | 15 +- .../{ => raw}/undeclared_component.py | 54 +- .../services/dependency_track_service.py | 131 +++ tests/test_policy_inspect.py | 296 +++++-- 19 files changed, 1686 insertions(+), 477 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/scanoss/inspection/dependency_track/project_violation.py rename src/scanoss/inspection/{ => raw}/component_summary.py (96%) rename src/scanoss/inspection/{ => raw}/copyleft.py (71%) rename src/scanoss/inspection/{ => raw}/license_summary.py (95%) rename src/scanoss/inspection/{inspect_base.py => raw/raw_base.py} (98%) rename src/scanoss/inspection/{ => raw}/undeclared_component.py (89%) create mode 100644 src/scanoss/services/dependency_track_service.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c2b1101c..e01c4812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.31.0] - 2025-08-08 +### Added +- Add `inspect dependency-track project-violations` subcommand to retrieve Dependency Track project violations in Markdown and JSON formats +### Changed +- Renamed `inspect copyleft` to `inspect raw copyleft` +- Renamed `inspect undeclared` to `inspect raw undeclared` +- Renamed `inspect component-summary` to `inspect raw component-summary` +- Renamed `inspect license-summary` to `inspect raw license-summary` +- Updated Policy return codes. 0 → Success, 2 → Fail, 1 → Error +### Fixed +- Fixed incorrect folder filtering configurations for fingerprinting and scanning + ## [1.30.0] - 2025-07-22 ### Added - Add `export dt` subcommand to export SBOM files to Dependency Track @@ -615,5 +627,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.28.2]: https://github.com/scanoss/scanoss.py/compare/v1.28.1...v1.28.2 [1.29.0]: https://github.com/scanoss/scanoss.py/compare/v1.28.2...v1.29.0 [1.30.0]: https://github.com/scanoss/scanoss.py/compare/v1.29.0...v1.30.0 - +[1.31.0]: https://github.com/scanoss/scanoss.py/compare/v1.30.0...v1.31.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 12e2f164..e6f3e865 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -367,10 +367,11 @@ The `inspect` command has a suite of sub-commands designed to inspect the result Details, such as license compliance or component declarations, can be examined. For example: -* Copyleft (`copylefet`) +* Copyleft (`copyleft`) * Undeclared Components (`undeclared`) * License Summary (`license-summary`) * Component Summary (`component-summary`) +* Dependency Track project violations (`dependency-track project-violations`) For the latest list of sub-commands, please run: ```bash @@ -476,6 +477,21 @@ Example with an output file: scanoss-py insp component-summary -i scan-results.json --output component-summary.json ``` +#### Inspect Dependency Track project violations Markdown output +The following command can be used to retrieve project violations from Dependency Track in Markdown format. + +**Note:** The upload token is optional. It is used to check the project processing status. If no token is provided, the latest project violations will be retrieved without waiting for project processing to complete. + +Example with project id: +```bash +scanoss-py inspect dt project-violations --dt-upload-token --dt-url --dt-projectid --dt-apikey --format md --output project-violations.md +``` +Example with project name and version: +```bash +scanoss-py inspect dt project-violations --dt-upload-token --dt-url --dt-projectname --dt-projectversion --dt-apikey --format md --output project-violations.md +``` + + ### Folder-Scan a Project Folder The new `folder-scan` subcommand performs a comprehensive scan on an entire directory by recursively processing files to generate folder-level fingerprints. It computes CRC64 hashes and simhash values to detect directory-level similarities, which is especially useful for comparing large code bases or detecting duplicate folder structures. diff --git a/Dockerfile b/Dockerfile index 063fd113..14d18c2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ FROM base AS builder # Setup the required build tooling RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential gcc \ + && apt-get install -y --no-install-recommends build-essential gcc libicu-dev pkg-config \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -54,9 +54,9 @@ COPY --from=builder /opt/venv /opt/venv ENV PATH=/opt/venv/bin:$PATH ENV GRPC_POLL_STRATEGY=poll -# Install jq and curl commands +# Install jq and curl commands and ICU runtime library RUN apt-get update \ - && apt-get install -y --no-install-recommends jq curl \ + && apt-get install -y --no-install-recommends jq curl libicu72 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/docs/source/index.rst b/docs/source/index.rst index 6bd8667c..dd774c4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -156,7 +156,7 @@ Calculates hashes for a directory or file and shows them on the STDOUT. - Fingerprint all hidden files/folders ----------------------------------------- -Detect dependecies: dependencies, dp, dep +Detect dependencies: dependencies, dp, dep ----------------------------------------- Scan source code for dependencies, but do not decorate them. diff --git a/pyproject.toml b/pyproject.toml index ccca9080..0925b1e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [tool.ruff] # Enable pycodestyle (E), pyflakes (F), isort (I), pylint (PL) -select = ["E", "F", "I", "PL"] +lint.select = ["E", "F", "I", "PL"] line-length = 120 # Assume Python 3.9+ target-version = "py39" @@ -22,3 +22,6 @@ line-ending = "auto" [tool.ruff.lint.isort] known-first-party = ["scanoss"] + +[tool.ruff.lint.pylint] +max-args = 5 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index eaa95be6..1868ffa4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.30.0' +__version__ = '1.31.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index a408d80f..057eedc3 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -33,12 +33,10 @@ import pypac from scanoss.cryptography import Cryptography, create_cryptography_config_from_args -from scanoss.export.dependency_track import ( - DependencyTrackExporter, - create_dependency_track_exporter_config_from_args, -) -from scanoss.inspection.component_summary import ComponentSummary -from scanoss.inspection.license_summary import LicenseSummary +from scanoss.export.dependency_track import DependencyTrackExporter +from scanoss.inspection.dependency_track.project_violation import DependencyTrackProjectViolationPolicyCheck +from scanoss.inspection.raw.component_summary import ComponentSummary +from scanoss.inspection.raw.license_summary import LicenseSummary from scanoss.scanners.container_scanner import ( DEFAULT_SYFT_COMMAND, DEFAULT_SYFT_TIMEOUT, @@ -69,8 +67,8 @@ from .csvoutput import CsvOutput from .cyclonedx import CycloneDx from .filecount import FileCount -from .inspection.copyleft import Copyleft -from .inspection.undeclared_component import UndeclaredComponent +from .inspection.raw.copyleft import Copyleft +from .inspection.raw.undeclared_component import UndeclaredComponent from .results import Results from .scancodedeps import ScancodeDeps from .scanner import FAST_WINNOWING, Scanner @@ -539,80 +537,293 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) p_results.set_defaults(func=results) - ########################################### INSPECT SUBCOMMAND ########################################### - # Sub-command: inspect + # ========================================================================= + # INSPECT SUBCOMMAND - Analysis and validation of scan results + # ========================================================================= + + # Main inspect parser - provides tools for analyzing scan results p_inspect = subparsers.add_parser( - 'inspect', aliases=['insp', 'ins'], description=f'Inspect results: {__version__}', help='Inspect results' + 'inspect', + aliases=['insp', 'ins'], + description=f'Inspect and analyse scan results: {__version__}', + help='Inspect and analyse scan results' ) - # Sub-parser: inspect + + # Inspect sub-commands parser p_inspect_sub = p_inspect.add_subparsers( - title='Inspect Commands', dest='subparsercmd', description='Inspect sub-commands', help='Inspect sub-commands' + title='Inspect Commands', + dest='subparsercmd', + description='Available inspection sub-commands', + help='Choose an inspection type' + ) + + # ------------------------------------------------------------------------- + # RAW RESULTS INSPECTION - Analyse raw scan output + # ------------------------------------------------------------------------- + + # Raw results parser - handles inspection of unprocessed scan results + p_inspect_raw = p_inspect_sub.add_parser( + 'raw', + description='Inspect and analyse SCANOSS raw scan results', + help='Analyse raw scan results for various compliance issues' + ) + + # Raw results sub-commands parser + p_inspect_raw_sub = p_inspect_raw.add_subparsers( + title='Raw Results Inspection Commands', + dest='subparser_subcmd', + description='Tools for analyzing raw scan results', + help='Choose a raw results analysis type' ) - ####### INSPECT: Copyleft ###### - # Inspect Sub-command: inspect copyleft - p_copyleft = p_inspect_sub.add_parser( - 'copyleft', aliases=['cp'], description='Inspect for copyleft licenses', help='Inspect for copyleft licenses' + # Copyleft license inspection - identifies copyleft license violations + p_inspect_raw_copyleft = p_inspect_raw_sub.add_parser( + 'copyleft', + aliases=['cp'], + description='Identify components with copyleft licenses that may require compliance action', + help='Find copyleft license violations' ) - ####### INSPECT: License Summary ###### - # Inspect Sub-command: inspect license summary - p_license_summary = p_inspect_sub.add_parser( + # License summary inspection - provides overview of all detected licenses + p_inspect_raw_license_summary = p_inspect_raw_sub.add_parser( 'license-summary', aliases=['lic-summary', 'licsum'], - description='Get license summary', - help='Get detected license summary from scan results', + description='Generate comprehensive summary of all licenses found in scan results', + help='Generate license summary report' ) - p_component_summary = p_inspect_sub.add_parser( + # Component summary inspection - provides overview of all detected components + p_inspect_raw_component_summary = p_inspect_raw_sub.add_parser( 'component-summary', aliases=['comp-summary', 'compsum'], - description='Get component summary', - help='Get detected component summary from scan results', + description='Generate comprehensive summary of all components found in scan results', + help='Generate component summary report' + ) + + # Undeclared components inspection - finds components not declared in SBOM + p_inspect_raw_undeclared = p_inspect_raw_sub.add_parser( + 'undeclared', + aliases=['un'], + description='Identify components present in code but not declared in SBOM files', + help='Find undeclared components' + ) + # SBOM format option for undeclared components inspection + p_inspect_raw_undeclared.add_argument( + '--sbom-format', + required=False, + choices=['legacy', 'settings'], + default='settings', + help='SBOM format type for comparison: legacy or settings (default)' ) - ####### INSPECT: Undeclared components ###### - # Inspect Sub-command: inspect undeclared - p_undeclared = p_inspect_sub.add_parser( + # ------------------------------------------------------------------------- + # BACKWARD COMPATIBILITY - Support old inspect command format + # ------------------------------------------------------------------------- + + # Legacy copyleft inspection - backward compatibility for 'scanoss-py inspect copyleft' + p_inspect_legacy_copyleft = p_inspect_sub.add_parser( + 'copyleft', + aliases=['cp'], + description='Identify components with copyleft licenses that may require compliance action', + help='Find copyleft license violations (legacy format)' + ) + + # Legacy undeclared components inspection - backward compatibility for 'scanoss-py inspect undeclared' + p_inspect_legacy_undeclared = p_inspect_sub.add_parser( 'undeclared', aliases=['un'], - description='Inspect for undeclared components', - help='Inspect for undeclared components', + description='Identify components present in code but not declared in SBOM files', + help='Find undeclared components (legacy format)' ) - p_undeclared.add_argument( + + # SBOM format option for legacy undeclared components inspection + p_inspect_legacy_undeclared.add_argument( '--sbom-format', required=False, choices=['legacy', 'settings'], default='settings', - help='Sbom format for status output', + help='SBOM format type for comparison: legacy or settings (default)' + ) + + # Legacy license summary inspection - backward compatibility for 'scanoss-py inspect license-summary' + p_inspect_legacy_license_summary = p_inspect_sub.add_parser( + 'license-summary', + aliases=['lic-summary', 'licsum'], + description='Generate comprehensive summary of all licenses found in scan results', + help='Generate license summary report (legacy format)' + ) + + # Legacy component summary inspection - backward compatibility for 'scanoss-py inspect component-summary' + p_inspect_legacy_component_summary = p_inspect_sub.add_parser( + 'component-summary', + aliases=['comp-summary', 'compsum'], + description='Generate comprehensive summary of all components found in scan results', + help='Generate component summary report (legacy format)' ) - # Add common commands for inspect copyleft and license summary - for p in [p_copyleft, p_license_summary]: + # Applies the same configuration to both legacy and raw versions + # License filtering options - common to (legacy) copyleft and license summary commands + for p in [p_inspect_raw_copyleft, p_inspect_raw_license_summary, + p_inspect_legacy_copyleft, p_inspect_legacy_license_summary]: p.add_argument( '--include', - help='List of Copyleft licenses to append to the default list. Provide licenses as a comma-separated list.', + help='Additional licenses to include in analysis (comma-separated list)' ) p.add_argument( '--exclude', - help='List of Copyleft licenses to remove from default list. Provide licenses as a comma-separated list.', + help='Licenses to exclude from analysis (comma-separated list)' ) p.add_argument( '--explicit', - help='Explicit list of Copyleft licenses to consider. Provide licenses as a comma-separated list.s', + help='Use only these specific licenses for analysis (comma-separated list)' ) - # Add common commands for inspect copyleft and license summary - for p in [p_license_summary, p_component_summary]: - p.add_argument('-i', '--input', nargs='?', help='Path to results file') - p.add_argument('-o', '--output', type=str, help='Save summary into a file') + # Common options for (legacy) copyleft and undeclared component inspection + for p in [p_inspect_raw_copyleft, p_inspect_raw_undeclared, p_inspect_legacy_copyleft, p_inspect_legacy_undeclared]: + p.add_argument( + '-i', '--input', + nargs='?', + help='Path to scan results file to analyse' + ) + p.add_argument( + '-f', '--format', + required=False, + choices=['json', 'md', 'jira_md'], + default='json', + help='Output format: json (default), md (Markdown), or jira_md (JIRA Markdown)' + ) + p.add_argument( + '-o', '--output', + type=str, + help='Save detailed results to specified file' + ) + p.add_argument( + '-s', '--status', + type=str, + help='Save summary status report to Markdown file' + ) - p_undeclared.set_defaults(func=inspect_undeclared) - p_copyleft.set_defaults(func=inspect_copyleft) - p_license_summary.set_defaults(func=inspect_license_summary) - p_component_summary.set_defaults(func=inspect_component_summary) + # Common options for (legacy) license and component summary commands + for p in [p_inspect_raw_license_summary, p_inspect_raw_component_summary, + p_inspect_legacy_license_summary, p_inspect_legacy_component_summary]: + p.add_argument( + '-i', '--input', + nargs='?', + help='Path to scan results file to analyse' + ) + p.add_argument( + '-o', '--output', + type=str, + help='Save summary report to specified file' + ) - ########################################### END INSPECT SUBCOMMAND ########################################### + # ------------------------------------------------------------------------- + # DEPENDENCY TRACK INSPECTION - Analyse Dependency Track project data + # ------------------------------------------------------------------------- + + # Dependency Track parser - handles inspection of DT project status and violations + p_dep_track_sub = p_inspect_sub.add_parser( + 'dependency-track', + aliases=['dt'], + description='Inspect and analyse Dependency Track project status and policy violations', + help='Analyse Dependency Track projects' + ) + + # Dependency Track sub-commands parser + p_inspect_dep_track_sub = p_dep_track_sub.add_subparsers( + title='Dependency Track Inspection Commands', + dest='subparser_subcmd', + description='Tools for analysing Dependency Track project data', + help='Choose a Dependency Track analysis type' + ) + + # Project violations inspection - analyses policy violations in DT projects + p_inspect_dt_project_violation = p_inspect_dep_track_sub.add_parser( + 'project-violations', + aliases=['pv'], + description='Analyse policy violations and compliance issues in Dependency Track projects', + help='Inspect project policy violations' + ) + # Dependency Track connection and authentication options + p_inspect_dt_project_violation.add_argument( + '--url', + required=True, + type=str, + help='Dependency Track server base URL (e.g., https://dtrack.example.com)' + ) + p_inspect_dt_project_violation.add_argument( + '--upload-token', '-ut', + required=False, + type=str, + help='Project-specific upload token for accessing DT project data' + ) + p_inspect_dt_project_violation.add_argument( + '--project-id', '-pid', + required=False, + type=str, + help='Dependency Track project UUID to inspect' + ) + p_inspect_dt_project_violation.add_argument( + '--apikey', '-k', + required=True, + type=str, + help='Dependency Track API key for authentication' + ) + p_inspect_dt_project_violation.add_argument( + '--project-name', '-pn', + required=False, + type=str, + help='Dependency Track project name' + ) + p_inspect_dt_project_violation.add_argument( + '--project-version', '-pv', + required=False, + type=str, + help='Dependency Track project version' + ) + p_inspect_dt_project_violation.add_argument( + '--output', '-o', + required=False, + type=str, + help='Save inspection results to specified file' + ) + p_inspect_dt_project_violation.add_argument( + '--status', + required=False, + type=str, + help='Save summary status report to specified file' + ) + p_inspect_dt_project_violation.add_argument( + '--format', '-f', + required=False, + choices=['json', 'md'], + default='json', + help='Output format: json (default) or md (Markdown)' + ) + p_inspect_dt_project_violation.add_argument( + '--timeout', '-M', + required=False, + default='300', + help='Timeout (in seconds) for API communication (optional - default 300 sec)' + ) + + # TODO Move to the command call def location + # RAW results + p_inspect_raw_undeclared.set_defaults(func=inspect_undeclared) + p_inspect_raw_copyleft.set_defaults(func=inspect_copyleft) + p_inspect_raw_license_summary.set_defaults(func=inspect_license_summary) + p_inspect_raw_component_summary.set_defaults(func=inspect_component_summary) + # Legacy backward compatibility commands + p_inspect_legacy_copyleft.set_defaults(func=inspect_copyleft) + p_inspect_legacy_undeclared.set_defaults(func=inspect_undeclared) + p_inspect_legacy_license_summary.set_defaults(func=inspect_license_summary) + p_inspect_legacy_component_summary.set_defaults(func=inspect_component_summary) + # Dependency Track + p_inspect_dt_project_violation.set_defaults(func=inspect_dep_track_project_violations) + + # ========================================================================= + # END INSPECT SUBCOMMAND CONFIGURATION + # ========================================================================= # Sub-command: export p_export = subparsers.add_parser( @@ -637,11 +848,12 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Upload SBOM files to Dependency Track', ) e_dt.add_argument('-i', '--input', type=str, required=True, help='Input SBOM file (CycloneDX JSON format)') - e_dt.add_argument('--dt-url', type=str, required=True, help='Dependency Track base URL') - e_dt.add_argument('--dt-apikey', type=str, required=True, help='Dependency Track API key') - e_dt.add_argument('--dt-projectid', type=str, help='Dependency Track project UUID') - e_dt.add_argument('--dt-projectname', type=str, help='Dependency Track project name') - e_dt.add_argument('--dt-projectversion', type=str, help='Dependency Track project version') + e_dt.add_argument('--url', type=str, required=True, help='Dependency Track base URL') + e_dt.add_argument('--apikey', '-k', type=str, required=True, help='Dependency Track API key') + e_dt.add_argument('--output', '-o', type=str, help='File to save export token and uuid into') + e_dt.add_argument('--project-id', '-pid', type=str, help='Dependency Track project UUID') + e_dt.add_argument('--project-name', '-pn', type=str, help='Dependency Track project name') + e_dt.add_argument('--project-version', '-pv', type=str, help='Dependency Track project version') e_dt.set_defaults(func=export_dt) # Sub-command: folder-scan @@ -746,19 +958,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Skip default settings file (scanoss.json) if it exists', ) - for p in [p_copyleft, p_undeclared]: - p.add_argument('-i', '--input', nargs='?', help='Path to results file') - p.add_argument( - '-f', - '--format', - required=False, - choices=['json', 'md', 'jira_md'], - default='json', - help='Output format (default: json)', - ) - p.add_argument('-o', '--output', type=str, help='Save details into a file') - p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file') - # Global Scan command options for p in [p_scan, p_cs]: p.add_argument( @@ -886,10 +1085,15 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_versions, c_semgrep, p_results, - p_undeclared, - p_copyleft, - p_license_summary, - p_component_summary, + p_inspect_raw_undeclared, + p_inspect_raw_copyleft, + p_inspect_raw_license_summary, + p_inspect_raw_component_summary, + p_inspect_legacy_copyleft, + p_inspect_legacy_undeclared, + p_inspect_legacy_license_summary, + p_inspect_legacy_component_summary, + p_inspect_dt_project_violation, c_provenance, p_folder_scan, p_folder_hash, @@ -916,6 +1120,9 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed sys.exit(1) + elif (args.subparser in 'inspect') and (args.subparsercmd in ('raw', 'dt')) and (args.subparser_subcmd is None): + parser.parse_args([args.subparser, args.subparsercmd, '--help']) # Force utils helps to be displayed + sys.exit(1) args.func(parser, args) # Execute the function associated with the sub-command @@ -949,16 +1156,14 @@ def file_count(parser, args): print_stderr('Please specify a folder') parser.parse_args([args.subparser, '-h']) sys.exit(1) - scan_output: str = None if args.output: - scan_output = args.output - open(scan_output, 'w').close() + initialise_empty_file(args.output) counter = FileCount( debug=args.debug, quiet=args.quiet, trace=args.trace, - scan_output=scan_output, + scan_output=args.output, hidden_files_folders=args.all_hidden, ) if not os.path.exists(args.scan_dir): @@ -987,10 +1192,8 @@ def wfp(parser, args): sys.exit(1) if args.strip_hpsm and not args.hpsm and not args.quiet: print_stderr('Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') - scan_output: str = None if args.output: - scan_output = args.output - open(scan_output, 'w').close() + initialise_empty_file(args.output) # Load scan settings scan_settings = None @@ -1023,15 +1226,15 @@ def wfp(parser, args): ) if args.stdin: contents = sys.stdin.buffer.read() - scanner.wfp_contents(args.stdin, contents, scan_output) + scanner.wfp_contents(args.stdin, contents, args.output) elif args.scan_dir: if not os.path.exists(args.scan_dir): print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.') sys.exit(1) if os.path.isdir(args.scan_dir): - scanner.wfp_folder(args.scan_dir, scan_output) + scanner.wfp_folder(args.scan_dir, args.output) elif os.path.isfile(args.scan_dir): - scanner.wfp_file(args.scan_dir, scan_output) + scanner.wfp_file(args.scan_dir, args.output) else: print_stderr(f'Error: Path specified is neither a file or a folder: {args.scan_dir}.') sys.exit(1) @@ -1128,10 +1331,8 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 if args.strip_hpsm and not args.hpsm and not args.quiet: print_stderr('Warning: --strip-hpsm option supplied without enabling HPSM (--hpsm). Ignoring.') - scan_output: str = None if args.output: - scan_output = args.output - open(scan_output, 'w').close() + initialise_empty_file(args.output) output_format = args.format if args.format else 'plain' flags = args.flags if args.flags else None if args.debug and not args.quiet: @@ -1186,7 +1387,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 quiet=args.quiet, api_key=args.key, url=args.apiurl, - scan_output=scan_output, + scan_output=args.output, output_format=output_format, flags=flags, nb_threads=args.threads, @@ -1298,16 +1499,15 @@ def dependency(parser, args): if not os.path.exists(args.scan_loc): print_stderr(f'Error: File or folder specified does not exist: {args.scan_loc}.') sys.exit(1) - scan_output: str = None if args.output: - scan_output = args.output - open(scan_output, 'w').close() + initialise_empty_file(args.output) sc_deps = ScancodeDeps( debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout ) - if not sc_deps.get_dependencies(what_to_scan=args.scan_loc, result_output=scan_output): + if not sc_deps.get_dependencies(what_to_scan=args.scan_loc, result_output=args.output): sys.exit(1) + return None def convert(parser, args): @@ -1346,179 +1546,340 @@ def convert(parser, args): sys.exit(1) -################################ INSPECT handlers ################################ +# ============================================================================= +# INSPECT COMMAND HANDLERS - Functions that execute inspection operations +# ============================================================================= + def inspect_copyleft(parser, args): """ - Run the "inspect" sub-command + Handle copyleft license inspection command. + + Analyses scan results to identify components using copyleft licenses + that may require compliance actions such as source code disclosure. + Parameters ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments - """ + parser : ArgumentParser + Command line parser object for help display + args : Namespace + Parsed command line arguments containing: + - input: Path to scan results file + - output: Optional output file path + - status: Optional status summary file path + - format: Output format (json, md, jira_md) + - include/exclude/explicit: License filter options + """ + # Validate required input file parameter if args.input is None: - print_stderr('Please specify an input file to inspect') - parser.parse_args([args.subparser, args.subparsercmd, '-h']) + print_stderr('ERROR: Input file is required for copyleft inspection') + parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) sys.exit(1) - output: str = None + # Initialise output file if specified if args.output: - output = args.output - open(output, 'w').close() - - status_output: str = None + initialise_empty_file(args.output) + # Initialise status summary file if specified if args.status: - status_output = args.status - open(status_output, 'w').close() + initialise_empty_file(args.status) + try: + # Create and configure copyleft inspector + i_copyleft = Copyleft( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + filepath=args.input, + format_type=args.format, + status=args.status, + output=args.output, + include=args.include, # Additional licenses to check + exclude=args.exclude, # Licenses to ignore + explicit=args.explicit, # Explicit license list + ) - i_copyleft = Copyleft( - debug=args.debug, - trace=args.trace, - quiet=args.quiet, - filepath=args.input, - format_type=args.format, - status=status_output, - output=output, - include=args.include, - exclude=args.exclude, - explicit=args.explicit, - ) - status, _ = i_copyleft.run() - sys.exit(status) + # Execute inspection and exit with appropriate status code + status, _ = i_copyleft.run() + sys.exit(status) + except Exception as e: + print_stderr(e) + if args.debug: + traceback.print_exc() + sys.exit(1) def inspect_undeclared(parser, args): """ - Run the "inspect" sub-command + Handle undeclared components inspection command. + + Analyses scan results to identify components that are present in the + codebase but not declared in SBOM or manifest files, which may indicate + security or compliance risks. + Parameters ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments - """ + parser : ArgumentParser + Command line parser object for help display + args : Namespace + Parsed command line arguments containing: + - input: Path to scan results file + - output: Optional output file path + - status: Optional status summary file path + - format: Output format (json, md, jira_md) + - sbom_format: SBOM format type (legacy, settings) + """ + # Validate required input file parameter if args.input is None: - print_stderr('Please specify an input file to inspect') - parser.parse_args([args.subparser, args.subparsercmd, '-h']) + print_stderr('ERROR: Input file is required for undeclared component inspection') + parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) sys.exit(1) - output: str = None + + # Initialise output file if specified if args.output: - output = args.output - open(output, 'w').close() + initialise_empty_file(args.output) - status_output: str = None + # Initialise status summary file if specified if args.status: - status_output = args.status - open(status_output, 'w').close() - i_undeclared = UndeclaredComponent( - debug=args.debug, - trace=args.trace, - quiet=args.quiet, - filepath=args.input, - format_type=args.format, - status=status_output, - output=output, - sbom_format=args.sbom_format, - ) - status, _ = i_undeclared.run() - sys.exit(status) + initialise_empty_file(args.status) + + try: + # Create and configure undeclared component inspector + i_undeclared = UndeclaredComponent( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + filepath=args.input, + format_type=args.format, + status=args.status, + output=args.output, + sbom_format=args.sbom_format, # Format for SBOM comparison + ) + + # Execute inspection and exit with appropriate status code + status, _ = i_undeclared.run() + sys.exit(status) + except Exception as e: + print_stderr(e) + if args.debug: + traceback.print_exc() + sys.exit(1) def inspect_license_summary(parser, args): """ - Run the "inspect" sub-command + Handle license summary inspection command. + + Generates comprehensive summary of all licenses detected in scan results, + including license counts, risk levels, and compliance recommendations. + Parameters ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments - """ + parser : ArgumentParser + Command line parser object for help display + args : Namespace + Parsed command line arguments containing: + - input: Path to scan results file + - output: Optional output file path + - include/exclude/explicit: License filter options + """ + # Validate required input file parameter if args.input is None: - print_stderr('Please specify an input file to inspect') - parser.parse_args([args.subparser, args.subparsercmd, '-h']) + print_stderr('ERROR: Input file is required for license summary') + parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) sys.exit(1) - output: str = None + + # Initialise output file if specified if args.output: - output = args.output - open(output, 'w').close() + initialise_empty_file(args.output) + # Create and configure license summary generator i_license_summary = LicenseSummary( debug=args.debug, trace=args.trace, quiet=args.quiet, filepath=args.input, - output=output, - include=args.include, - exclude=args.exclude, - explicit=args.explicit, + output=args.output, + include=args.include, # Additional licenses to include + exclude=args.exclude, # Licenses to exclude from summary + explicit=args.explicit, # Explicit license list to summarize ) - i_license_summary.run() - + try: + # Execute summary generation + i_license_summary.run() + except Exception as e: + print_stderr(e) + if args.debug: + traceback.print_exc() + sys.exit(1) def inspect_component_summary(parser, args): """ - Run the "inspect" sub-command + Handle component summary inspection command. + + Generates a comprehensive summary of all components detected in scan results, + including component counts, versions, match types, and security information. + Parameters ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments - """ + parser : ArgumentParser + Command line parser object for help display + args : Namespace + Parsed command line arguments containing: + - input: Path to scan results file + - output: Optional output file path + """ + # Validate required input file parameter if args.input is None: - print_stderr('Please specify an input file to inspect') - parser.parse_args([args.subparser, args.subparsercmd, '-h']) + print_stderr('ERROR: Input file is required for component summary') + parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) sys.exit(1) - output: str = None + + # Initialise an output file if specified if args.output: - output = args.output - open(output, 'w').close() + initialise_empty_file(args.output) # Create/clear output file + # Create and configure component summary generator i_component_summary = ComponentSummary( debug=args.debug, trace=args.trace, quiet=args.quiet, filepath=args.input, - output=output, + output=args.output, ) - i_component_summary.run() - - -################################ End inspect handlers ################################ + try: + # Execute summary generation + i_component_summary.run() + except Exception as e: + print_stderr(e) + if args.debug: + traceback.print_exc() + sys.exit(1) -def export_dt(parser, args): +def inspect_dep_track_project_violations(parser, args): """ - Run the "export dt" sub-command + Handle Dependency Track project inspection command. + + Analyses Dependency Track projects for policy violations, security issues, + and compliance status. Connects to DT API to retrieve project data and + generate detailed violation reports. + Parameters ---------- - parser: ArgumentParser - command line parser object - args: Namespace - Parsed arguments + parser : ArgumentParser + Command line parser object for help display + args : Namespace + Parsed command line arguments containing: + - url: Dependency Track base URL + - apikey: API key for authentication + - project_id: Project UUID to inspect + - project_name: Project name to inspect + - project_version: Project version to inspect + - upload_token: Upload token for project access + - output: Optional output file path + - format: Output format (json, md) + - timeout: Optional timeout for API requests + + """ + # Make sure we have project id/project name and version + _dt_args_validator(parser, args) + # Initialise the output file if specified + if args.output: + initialise_empty_file(args.output) + # Create and configure Dependency Track inspector + try: + dt_proj_violations = DependencyTrackProjectViolationPolicyCheck( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + output=args.output, + status= args.status, + format_type=args.format, + url=args.url, # DT server URL + api_key=args.apikey, # Authentication key + project_id=args.project_id, # Target project UUID + upload_token=args.upload_token, # Upload access token + project_name=args.project_name, # DT project name + project_version=args.project_version, # DT project version + timeout=args.timeout, + ) + + # Execute inspection and exit with appropriate status code + status, _ = dt_proj_violations.run() #TODO remove datastructure from return + sys.exit(status) + except Exception as e: + print_stderr(e) + if args.debug: + traceback.print_exc() + sys.exit(1) + + +# ============================================================================= +# END INSPECT COMMAND HANDLERS +# ============================================================================= + +def export_dt(parser, args): """ + Validates and exports a Software Bill of Materials (SBOM) to a Dependency-Track server. + + Parameters: + parser (argparse.ArgumentParser): The argument parser to validate input arguments. + args (argparse.Namespace): Parsed arguments passed to the command. + Raises: + SystemExit: If argument validation fails or uploading the SBOM to the Dependency-Track server + is unsuccessful. + """ + # Make sure we have project id/project name and version + _dt_args_validator(parser, args) + if args.output: + initialise_empty_file(args.output) + if not args.quiet: + print_stderr(f'Outputting export data result to: {args.output}') try: - config = create_dependency_track_exporter_config_from_args(args) dt_exporter = DependencyTrackExporter( - config=config, + url=args.url, + apikey=args.apikey, + output=args.output, debug=args.debug, trace=args.trace, quiet=args.quiet, ) - - success = dt_exporter.upload_sbom(args.input) - + success = dt_exporter.upload_sbom_file(args.input, args.project_id, args.project_name, + args.project_version, args.output) if not success: sys.exit(1) - except Exception as e: print_stderr(f'ERROR: {e}') if args.debug: traceback.print_exc() sys.exit(1) +def _dt_args_validator(parser, args): + """ + Validates command-line arguments related to project identification. + + Parameters + ---------- + parser : argparse.ArgumentParser + An argument parser instance for handling command-line arguments. + args : argparse.Namespace + Parsed arguments from the command line containing project-related information. + + Raises + ------ + SystemExit + If neither a project ID nor the required combination of project name and + project version is provided, or if any of the compulsory arguments + are missing. + """ + if not args.project_id and not args.project_name and not args.project_version: + print_stderr( + 'Please specify either a project ID (--project-id) or a project name (--project-name) and ' + 'version (--project-version)' + ) + parser.parse_args([args.subparser, '-h']) + sys.exit(1) + if not args.project_id and (not args.project_name or not args.project_version): + print_stderr('Please supply a project name (--project-name) and version (--project-version)') + sys.exit(1) def utils_certloc(*_): """ @@ -2123,7 +2484,31 @@ def get_scanoss_settings_from_args(args): except ScanossSettingsError as e: print_stderr(f'Error: {e}') sys.exit(1) - return scanoss_settings + return scanoss_settings + + +def initialise_empty_file(filename: str): + """ + Initialises an empty file with the specified name. If the file already exists, + it truncates its content. Ensures proper error handling in case of failure. + + Args: + filename (str): The name of the file to be initialised. + + Raises: + SystemExit: If the file cannot be created or written due to an exception, + the function prints an error message and exits the program. + + Note: + This function writes an empty file and handles exceptions to ensure the + program does not continue execution in case of an error. + """ + if filename: + try: + open(filename, 'w').close() + except Exception as e: + print_stderr(f'Error: Unable to create output file {filename}: {e}') + sys.exit(1) def main(): diff --git a/src/scanoss/export/dependency_track.py b/src/scanoss/export/dependency_track.py index acaeb623..38dccd4d 100644 --- a/src/scanoss/export/dependency_track.py +++ b/src/scanoss/export/dependency_track.py @@ -25,85 +25,65 @@ import base64 import json import traceback -from dataclasses import dataclass -from typing import Optional import requests -from scanoss.cyclonedx import CycloneDx - +from ..cyclonedx import CycloneDx from ..scanossbase import ScanossBase +from ..services.dependency_track_service import DependencyTrackService from ..utils.file import validate_json_file -@dataclass -class DependencyTrackExporterConfig: - debug: bool = False - trace: bool = False - quiet: bool = False - dt_url: str = None - dt_apikey: str = None - dt_projectid: Optional[str] = None - dt_projectname: Optional[str] = None - dt_projectversion: Optional[str] = None +def _build_payload(encoded_sbom: str, project_id, project_name, project_version) -> dict: + """ + Build the API payload + Args: + encoded_sbom: Base64 encoded SBOM -def create_dependency_track_exporter_config_from_args(args) -> DependencyTrackExporterConfig: - return DependencyTrackExporterConfig( - debug=getattr(args, 'debug', False), - trace=getattr(args, 'trace', False), - quiet=getattr(args, 'quiet', False), - dt_url=getattr(args, 'dt_url', None), - dt_apikey=getattr(args, 'dt_apikey', None), - dt_projectid=getattr(args, 'dt_projectid', None), - dt_projectname=getattr(args, 'dt_projectname', None), - dt_projectversion=getattr(args, 'dt_projectversion', None), - ) + Returns: + API payload dictionary + """ + if project_id: + return {'project': project_id, 'bom': encoded_sbom} + else: + return { + 'projectName': project_name, + 'projectVersion': project_version, + 'autoCreate': True, + 'bom': encoded_sbom, + } class DependencyTrackExporter(ScanossBase): """ Class for exporting SBOM files to Dependency Track """ - - def __init__( + def __init__( # noqa: PLR0913 self, - config: DependencyTrackExporterConfig, + url: str = None, + apikey: str = None, + output: str = None, debug: bool = False, trace: bool = False, - quiet: bool = False, + quiet: bool = False ): """ Initialize DependencyTrackExporter Args: - config: Configuration parameters for the dependency track exporter + url: Dependency Track URL + apikey: Dependency Track API Key + output: File to store output response data (optional) debug: Enable debug output trace: Enable trace output quiet: Enable quiet mode """ super().__init__(debug=debug, trace=trace, quiet=quiet) - - self.dt_url = config.dt_url.rstrip('/') - self.dt_apikey = config.dt_apikey - self.dt_projectid = config.dt_projectid - self.dt_projectname = config.dt_projectname - self.dt_projectversion = config.dt_projectversion - - self._validate_config() - - def _validate_config(self): - """ - Validate that the configuration is valid. - """ - has_id = bool(self.dt_projectid) - has_name_version = bool(self.dt_projectname and self.dt_projectversion) - - if not (has_id or has_name_version): - raise ValueError('Either --dt-projectid OR (--dt-projectname and --dt-projectversion) must be provided') - - if has_id and has_name_version: - self.print_debug('Both DT project ID and name/version provided. Using project ID.') + self.url = url.rstrip('/') + self.apikey = apikey + self.output = output + self.dt_service = DependencyTrackService(self.apikey, self.url, debug=debug, trace=trace, quiet=quiet) def _read_and_validate_sbom(self, input_file: str) -> dict: """ @@ -116,7 +96,7 @@ def _read_and_validate_sbom(self, input_file: str) -> dict: Parsed SBOM content as dictionary Raises: - ValueError: If file doesn't exist or is invalid or not a valid CycloneDX SBOM + ValueError: If the file doesn't exist or is invalid or not a valid CycloneDX SBOM """ result = validate_json_file(input_file) if not result.is_valid: @@ -125,7 +105,6 @@ def _read_and_validate_sbom(self, input_file: str) -> dict: cdx = CycloneDx(debug=self.debug) if not cdx.is_cyclonedx_json(json.dumps(result.data)): raise ValueError(f'Input file is not a valid CycloneDX SBOM: {input_file}') - return result.data def _encode_sbom(self, sbom_content: dict) -> str: @@ -138,84 +117,106 @@ def _encode_sbom(self, sbom_content: dict) -> str: Returns: Base64 encoded string """ + if not sbom_content: + self.print_stderr('Warning: Empty SBOM content') json_str = json.dumps(sbom_content, separators=(',', ':')) encoded = base64.b64encode(json_str.encode('utf-8')).decode('utf-8') return encoded - def _build_payload(self, encoded_sbom: str) -> dict: + def upload_sbom_file(self, input_file, project_id, project_name, project_version, output_file): """ - Build the API payload + Uploads an SBOM file to the specified project with an + optional output file and processes the file content for validation. Args: - encoded_sbom: Base64 encoded SBOM + input_file (str): The path to the SBOM file to be read and uploaded. + project_id (str): The unique identifier of the project to which the SBOM is being uploaded. + project_name (str): The name of the project to which the SBOM is being uploaded. + project_version (str): The version of the project to which the SBOM is being uploaded. + output_file (str): The path to save output related to the SBOM upload process. Returns: - API payload dictionary - """ - if self.dt_projectid: - return {'project': self.dt_projectid, 'bom': encoded_sbom} - else: - return { - 'projectName': self.dt_projectname, - 'projectVersion': self.dt_projectversion, - 'autoCreate': True, - 'bom': encoded_sbom, - } - - def upload_sbom(self, input_file: str) -> bool: - """ - Upload SBOM file to Dependency Track + bool: Returns True if the SBOM file was uploaded successfully, False otherwise. - Args: - input_file: Path to the SBOM file - - Returns: - True if successful, False otherwise + Raises: + ValueError: Raised if there are validation issues with the SBOM content. """ try: - self.print_stderr(f'Reading SBOM file: {input_file}') + if not self.quiet: + self.print_stderr(f'Reading SBOM file: {input_file}') sbom_content = self._read_and_validate_sbom(input_file) + return self.upload_sbom_contents(sbom_content, project_id, project_name, project_version, output_file) + except ValueError as e: + self.print_stderr(f'Validation error: {e}') + return False - self.print_debug('Encoding SBOM to base64') - encoded_sbom = self._encode_sbom(sbom_content) - - payload = self._build_payload(encoded_sbom) + def upload_sbom_contents(self, sbom_content: dict, project_id, project_name, project_version, output_file) -> bool: + """ + Uploads an SBOM to a Dependency Track server. - url = f'{self.dt_url}/api/v1/bom' - headers = {'Content-Type': 'application/json', 'X-Api-Key': self.dt_apikey} + Parameters: + sbom_content (dict): The SBOM content in dictionary format to be uploaded. + project_id: The unique identifier for the project. + project_name: The name of the project in Dependency Track. + project_version: The version of the project in Dependency Track. + output_file: The path to the file where the token and UUID data + should be written. If not provided, the data will be written to + standard output. - if self.trace: - self.print_trace(f'URL: {url}') - self.print_trace(f'Headers: {headers}') - self.print_trace(f'Payload keys: {list(payload.keys())}') + Returns: + bool: True if the upload is successful; False otherwise. + Raises: + ValueError: If the SBOM encoding process fails. + requests.exceptions.RequestException: If an error occurs during the HTTP request. + Exception: For any other unexpected error. + """ + if not project_id and not (project_name and project_version): + self.print_stderr('Error: Missing project id or name and version.') + return False + output = self.output + if output_file: + output = output_file + try: + self.print_debug('Encoding SBOM to base64') + payload = _build_payload(self._encode_sbom(sbom_content), project_id, project_name, project_version) + url = f'{self.url}/api/v1/bom' + headers = {'Content-Type': 'application/json', 'X-Api-Key': self.apikey} + self.print_trace(f'URL: {url}, Headers: {headers}, Payload keys: {list(payload.keys())}') self.print_msg('Uploading SBOM to Dependency Track...') response = requests.put(url, json=payload, headers=headers) - - if response.status_code in [200, 201]: - self.print_stderr('SBOM uploaded successfully') - + response.raise_for_status() + # Treat any 2xx status as success + if (requests.codes.ok <= response.status_code < requests.codes.multiple_choices and + response.status_code != requests.codes.no_content): + self.print_msg('SBOM uploaded successfully') try: response_data = response.json() + token = '' + project_uuid = project_id if 'token' in response_data: - self.print_stderr(f'Upload token: {response_data["token"]}') + token = response_data['token'] + if project_name and project_version: + project_data = self.dt_service.get_project_by_name_version(project_name, project_version) + if project_data: + project_uuid = project_data.get("uuid", project_id) + token_json = json.dumps( + {"token": token, "project_uuid": project_uuid}, + indent=2 + ) + self.print_to_file_or_stdout(token_json, output) except json.JSONDecodeError: pass - return True else: self.print_stderr(f'Upload failed with status code: {response.status_code}') self.print_stderr(f'Response: {response.text}') - return False - except ValueError as e: - self.print_stderr(f'Validation error: {e}') - return False + self.print_stderr(f'DT SBOM Upload Validation error: {e}') except requests.exceptions.RequestException as e: - self.print_stderr(f'Request error: {e}') - return False + self.print_stderr(f'DT API Request error: {e}') except Exception as e: self.print_stderr(f'Unexpected error: {e}') if self.debug: traceback.print_exc() - return False + return False diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index d52ea386..cb8298a8 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -72,11 +72,7 @@ 'htmlcov', '__pypackages__', 'example', - 'examples', - 'docs', - 'tests', - 'doc', - 'test', + 'examples' } DEFAULT_SKIPPED_DIRS_HFH = { diff --git a/src/scanoss/inspection/dependency_track/project_violation.py b/src/scanoss/inspection/dependency_track/project_violation.py new file mode 100644 index 00000000..0218d599 --- /dev/null +++ b/src/scanoss/inspection/dependency_track/project_violation.py @@ -0,0 +1,443 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import json +import time +from datetime import datetime +from typing import Any, Dict, List, Optional, TypedDict + +from ...services.dependency_track_service import DependencyTrackService +from ..policy_check import PolicyCheck, PolicyStatus + +# Constants +PROCESSING_RETRY_DELAY = 5 # seconds +DEFAULT_TIME_OUT = 300 +MILLISECONDS_TO_SECONDS = 1000 + + +""" +Dependency Track project violation policy check implementation. + +This module provides policy checking functionality for Dependency Track project violations. +It retrieves, processes, and formats policy violations from a Dependency Track instance +for a specific project. +""" + +class ResolvedLicenseDict(TypedDict): + """TypedDict for resolved license information from Dependency Track.""" + uuid: str + name: str + licenseId: str + isOsiApproved: bool + isFsfLibre: bool + isDeprecatedLicenseId: bool + isCustomLicense: bool + + +class ProjectDict(TypedDict): + """TypedDict for project information from Dependency Track.""" + authors: List[str] + name: str + version: str + classifier: str + collectionLogic: str + uuid: str + properties: List[Any] + tags: List[str] + lastBomImport: int + lastBomImportFormat: str + lastInheritedRiskScore: float + lastVulnerabilityAnalysis: int + active: bool + isLatest: bool + + +class ComponentDict(TypedDict): + """TypedDict for component information from Dependency Track.""" + authors: List[str] + name: str + version: str + classifier: str + purl: str + purlCoordinates: str + resolvedLicense: ResolvedLicenseDict + project: ProjectDict + lastInheritedRiskScore: float + uuid: str + expandDependencyGraph: bool + isInternal: bool + cpe: Optional[str] + + +class PolicyDict(TypedDict): + """TypedDict for policy information from Dependency Track.""" + name: str + operator: str + violationState: str + uuid: str + includeChildren: bool + onlyLatestProjectVersion: bool + + +class PolicyConditionDict(TypedDict): + """TypedDict for policy condition information from Dependency Track.""" + policy: PolicyDict + operator: str + subject: str + value: str + uuid: str + + +class PolicyViolationDict(TypedDict): + """TypedDict for policy violation information from Dependency Track.""" + type: str + project: ProjectDict + component: ComponentDict + policyCondition: PolicyConditionDict + timestamp: int + uuid: str + + +class DependencyTrackProjectViolationPolicyCheck(PolicyCheck[PolicyViolationDict]): + """ + Policy check implementation for Dependency Track project violations. + + This class handles retrieving, processing, and formatting policy violations + from a Dependency Track instance for a specific project. + """ + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + project_id: str = None, + project_name: str = None, + project_version: str = None, + api_key: str = None, + url: str = None, + upload_token: str = None, + timeout: float = DEFAULT_TIME_OUT, + format_type: str = None, + status: str = None, + output: str = None, + ): + """ + Initialise the Dependency Track project violation policy checker. + + Args: + debug: Enable debug output + trace: Enable trace output + quiet: Enable quiet mode + project_id: UUID of the project in Dependency Track + project_name: Name of the project in Dependency Track + project_version: Version of the project in Dependency Track + api_key: API key for Dependency Track authentication + url: Base URL of the Dependency Track instance + upload_token: Upload token for uploading BOMs to Dependency Track + format_type: Output format type (json, markdown, etc.) + status: Status output destination + output: Results output destination + timeout: Timeout for processing in seconds (default: 300) + """ + super().__init__(debug, trace, quiet, format_type, status, 'dependency-track', output) + self.url = url + self.api_key = api_key + self.project_id = project_id + self.project_name = project_name + self.project_version = project_version + self.upload_token = upload_token + self.timeout = timeout + self.dep_track_service = DependencyTrackService(self.api_key, self.url, debug=debug, trace=trace, quiet=quiet) + + def _json(self, project_violations: list[PolicyViolationDict]) -> Dict[str, Any]: + """ + Format project violations as JSON. + + Args: + project_violations: List of policy violations from Dependency Track + + Returns: + Dictionary containing JSON formatted results and summary + """ + return { + "details": json.dumps(project_violations, indent=2), + "summary": f'{len(project_violations)} policy violations were found.\n', + } + + def _markdown(self, project_violations: list[PolicyViolationDict]) -> Dict[str, Any]: + """ + Format Dependency Track violations to Markdown format. + + Args: + project_violations: List of policy violations from Dependency Track + + Returns: + Dictionary with formatted Markdown details and summary + """ + return self._md_summary_generator(project_violations, self.generate_table) + + def _jira_markdown(self, data: list[PolicyViolationDict]) -> Dict[str, Any]: + """ + Format project violations for Jira Markdown. + + Args: + data: List of policy violations from Dependency Track + + Returns: + Dictionary containing Jira markdown formatted results and summary + """ + return self._md_summary_generator(data, self.generate_jira_table) + + def is_project_updated(self, dt_project: Dict[str, Any]) -> bool: + """ + Check if a Dependency Track project has completed processing. + + This method determines if a project has finished processing by comparing + the timestamps of the last BOM import, vulnerability analysis, and last + occurrence metrics. A project is considered updated when either the + vulnerability analysis or the metrics' last occurrence timestamp is greater + than or equal to the last BOM import timestamp. + + Args: + dt_project: Project dictionary from Dependency Track containing + project metadata and timestamps + + Returns: + True if the project has completed processing (vulnerability analysis + or metrics are up to date with the last BOM import), False otherwise + """ + if not dt_project: + self.print_stderr('Warning: No project details supplied. Returning False.') + return False + last_import = dt_project.get('lastBomImport', 0) + last_vulnerability_analysis = dt_project.get('lastVulnerabilityAnalysis', 0) + metrics = dt_project.get('metrics', {}) + last_occurrence = metrics.get('lastOccurrence', 0) if isinstance(metrics, dict) else 0 + if self.debug: + self.print_msg(f'last_import: {last_import}') + self.print_msg(f'last_vulnerability_analysis: {last_vulnerability_analysis}') + self.print_msg(f'last_occurrence: {last_occurrence}') + self.print_msg(f'last_vulnerability_analysis is updated: {last_vulnerability_analysis >= last_import}') + self.print_msg(f'last_occurrence is updated: {last_occurrence >= last_import}') + if last_vulnerability_analysis == 0 or last_occurrence == 0 or last_import == 0: + self.print_stderr(f'Warning: Some project data appears to be unset. Returning False: {dt_project}') + return False + # True if: Both vulnerability analysis and metrics calculation newer than last BOM upload + return last_vulnerability_analysis >= last_import and last_occurrence >= last_import + + def _wait_processing_by_project_id(self) -> Optional[Any] or None: + """ + Wait for project processing to complete in Dependency Track. + + Returns: + Return the project or None if processing fails or times out + """ + start_time = time.time() + while True: + self.print_debug('Starting...') + dt_project = self.dep_track_service.get_project_by_id(self.project_id) + if not dt_project: + self.print_stderr(f'Failed to get project by id: {self.project_id}') + return None + is_project_updated = self.is_project_updated(dt_project) + if is_project_updated: # Project updated, return it + return dt_project + # Check timeout + if time.time() - start_time > self.timeout: + self.print_msg(f'Warning: Timeout reached ({self.timeout}s) while waiting for project processing') + return dt_project + time.sleep(PROCESSING_RETRY_DELAY) + self.print_debug('Checking if complete...') + # End while loop + + def _wait_processing_by_project_status(self): + """ + Wait for project processing to complete in Dependency Track. + + Returns: + Project status dictionary or None if processing fails or times out + """ + start_time = time.time() + while True: + status = self.dep_track_service.get_project_status(self.upload_token) + if status is None: + self.print_stderr(f'Error getting project status for upload token: {self.upload_token}') + break + if status and not status.get('processing'): + self.print_debug(f'Project Status: {status}') + break + if time.time() - start_time > self.timeout: + self.print_msg(f'Timeout reached ({self.timeout}s) while waiting for project processing') + break + time.sleep(PROCESSING_RETRY_DELAY) + self.print_debug('Checking if complete...') + # End while loop + + def _wait_project_processing(self): + """ + Wait for project processing to complete in Dependency Track. + + Returns: + Project status dictionary or None if processing fails + """ + if self.upload_token: + self.print_debug("Using upload token to check project status") + self._wait_processing_by_project_status() + self.print_debug("Using project id to get project status") + return self._wait_processing_by_project_id() + + def _set_project_id(self) -> None: + """ + Set the project ID based on the project name and version if not already set. + If no project id is specified, this method will attempt to retrieve the project based on name/version. + + Raises: + ValueError: If the project name/version is missing or the project is not found. + RuntimeError: If there's an error communicating with Dependency Track. + """ + if self.project_id is not None: + return + if self.project_name is None or self.project_version is None: + raise ValueError( + "Error: Project name and version must be specified when not using project ID" + ) + self.print_debug(f'Searching for project id by name and version: {self.project_name}@{self.project_version}') + dt_project = self.dep_track_service.get_project_by_name_version(self.project_name, self.project_version) + self.print_debug(f'dt_project: {dt_project}') + if dt_project is None: + raise ValueError(f'Error: Project {self.project_name}@{self.project_version} not found in Dependency Track') + self.project_id = dt_project.get('uuid') + if not self.project_id: + self.print_stderr(f'Error: Failed to get project uuid from: {dt_project}') + raise ValueError(f'Error: Project {self.project_name}@{self.project_version} does not have a valid UUID') + + @staticmethod + def _sort_project_violations(violations: List[PolicyViolationDict]) -> List[PolicyViolationDict]: + """ + Sort project violations by priority. + + Sorts violations with SECURITY issues first, followed by LICENSE, + then OTHER types. + + Args: + violations: List of policy violation dictionaries + + Returns: + Sorted list of policy violations + """ + type_priority = {'SECURITY': 3, 'LICENSE': 2, 'OTHER': 1} + return sorted( + violations, + key=lambda x: -type_priority.get(x.get('type', 'OTHER'), 1) + ) + + def _md_summary_generator(self, project_violations: list[PolicyViolationDict], table_generator): + """ + Generates a Markdown summary of project policy violations. + + Args: + project_violations (list[PolicyViolationDict]): A list of dictionaries containing details of + project policy violations, including violation state, risk type, policy name, component details, + and timestamp. + table_generator (function): A callable function responsible for generating the Markdown table + using headers, rows, and optionally highlighted columns. + + Returns: + dict: A dictionary with two keys: + - "details" containing a Markdown-compatible string with detailed project violations + rendered as a table + - "summary" summarising the number of violations found + """ + if project_violations is None: + self.print_stderr('Warning: No project violations found. Returning empty results.') + return { + "details": "h3. Dependency Track Project Violations\n\nNo policy violations found.\n", + "summary": "0 policy violations were found.\n", + } + headers = ['State', 'Risk Type', 'Policy Name', 'Component', 'Date'] + c_cols = [0, 1] + rows: List[List[str]] = [] + + for project_violation in project_violations: + timestamp = project_violation['timestamp'] + timestamp_seconds = timestamp / MILLISECONDS_TO_SECONDS + formatted_date = datetime.fromtimestamp(timestamp_seconds).strftime("%d %b %Y at %H:%M:%S") + + row = [ + project_violation['policyCondition']["policy"]["violationState"], + project_violation['type'], + project_violation['policyCondition']["policy"]["name"], + f'{project_violation["component"]["purl"]}@{project_violation["component"]["version"]}', + formatted_date, + ] + rows.append(row) + + return { + "details": f'### Dependency Track Project Violations\n{table_generator(headers, rows, c_cols)}\n', + "summary": f'{len(project_violations)} policy violations were found.\n', + } + + def run(self) -> int: + """ + Runs the primary execution logic of the instance. + + Returns: + int: Status code indicating the result of the run process. Possible + values are derived from the PolicyStatus enumeration. + FAIL if violations are found, SUCCESS if no violations are found, ERROR if an error occurs. + + Raises: + ValueError: If an invalid format is specified during the execution. + """ + # Set project ID based on name/version if needed + self._set_project_id() + if self.debug: + self.print_msg(f'URL: {self.url}') + self.print_msg(f'Project Id: {self.project_id}') + self.print_msg(f'Project Name: {self.project_name}') + self.print_msg(f'Project Version: {self.project_version}') + self.print_msg(f'API Key: {"*" * len(self.api_key)}') + self.print_msg(f'Format: {self.format_type}') + self.print_msg(f'Status: {self.status}') + self.print_msg(f'Output: {self.output}') + self.print_msg(f'Timeout: {self.timeout}') + # Confirm processing is complete before returning project violations + dt_project = self._wait_project_processing() + if not dt_project: + return PolicyStatus.ERROR.value + # Get project violations from Dependency Track + dt_project_violations = self.dep_track_service.get_project_violations(self.project_id) + # Sort violations by priority and format output + formatter = self._get_formatter() + if formatter is None: + self.print_stderr('Error: Invalid format specified.') + return PolicyStatus.ERROR.value + # Format and output data + data = formatter(self._sort_project_violations(dt_project_violations)) + self.print_to_file_or_stdout(data['details'], self.output) + self.print_to_file_or_stderr(data['summary'], self.status) + # Return appropriate status based on violation count + if len(dt_project_violations) > 0: + return PolicyStatus.POLICY_FAIL.value + return PolicyStatus.POLICY_SUCCESS.value diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index aba8185a..decde1aa 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -24,9 +24,9 @@ from abc import abstractmethod from enum import Enum -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, Generic, List, TypeVar -from .inspect_base import InspectBase +from ..scanossbase import ScanossBase from .utils.license_utils import LicenseUtil @@ -35,19 +35,20 @@ class PolicyStatus(Enum): Enumeration representing the status of a policy check. Attributes: - SUCCESS (int): Indicates that the policy check passed successfully (value: 0). - FAIL (int): Indicates that the policy check failed (value: 1). - ERROR (int): Indicates that an error occurred during the policy check (value: 2). + POLICY_SUCCESS (int): Indicates that the policy check passed successfully (value: 0). + POLICY_FAIL (int): Indicates that the policy check failed (value: 2). + ERROR (int): Indicates that an error occurred during the policy check (value: 1). """ - - SUCCESS = 0 - FAIL = 1 - ERROR = 2 + POLICY_SUCCESS = 0 + POLICY_FAIL = 2 + ERROR = 1 # # End of PolicyStatus Class # -class PolicyCheck(InspectBase): +T = TypeVar('T') + +class PolicyCheck(ScanossBase, Generic[T]): """ A base class for implementing various software policy checks. @@ -66,19 +67,17 @@ def __init__( # noqa: PLR0913 debug: bool = False, trace: bool = False, quiet: bool = False, - filepath: str = None, format_type: str = None, status: str = None, - output: str = None, name: str = None, + output: str = None, ): - super().__init__(debug, trace, quiet, filepath, output) + super().__init__(debug, trace, quiet) self.license_util = LicenseUtil() - self.filepath = filepath self.name = name self.format_type = format_type self.status = status - self.results = self._load_input_file() + self.output = output @abstractmethod def run(self): @@ -99,41 +98,41 @@ def run(self): pass @abstractmethod - def _json(self, components: list) -> Dict[str, Any]: + def _json(self, data: list[T]) -> Dict[str, Any]: """ Format the policy checks results as JSON. This method should be implemented by subclasses to create a Markdown representation of the policy check results. - :param components: List of components to be formatted. + :param data: List of data to be formatted. :return: A dictionary containing two keys: - - 'details': A JSON-formatted string with the full list of components + - 'results': A JSON-formatted string with the full list of components - 'summary': A string summarizing the number of components found """ pass @abstractmethod - def _markdown(self, components: list) -> Dict[str, Any]: + def _markdown(self, data: list[T]) -> Dict[str, Any]: """ Generate Markdown output for the policy check results. This method should be implemented by subclasses to create a Markdown representation of the policy check results. - :param components: List of components to be included in the output. + :param data: List of data to be included in the output. :return: A dictionary representing the Markdown output. """ pass @abstractmethod - def _jira_markdown(self, components: list) -> Dict[str, Any]: + def _jira_markdown(self, data: list[T]) -> Dict[str, Any]: """ Generate Markdown output for the policy check results. This method should be implemented by subclasses to create a Markdown representation of the policy check results. - :param components: List of components to be included in the output. + :param data: List of data to be included in the output. :return: A dictionary representing the Markdown output. """ pass @@ -208,7 +207,6 @@ def _debug(self): self.print_stderr(f'Format: {self.format_type}') self.print_stderr(f'Status: {self.status}') self.print_stderr(f'Output: {self.output}') - self.print_stderr(f'Input: {self.filepath}') def _is_valid_format(self) -> bool: """ @@ -224,6 +222,39 @@ def _is_valid_format(self) -> bool: self.print_stderr(f'ERROR: Invalid format "{self.format_type}". Valid formats are: {valid_formats_str}') return False return True + + def _generate_formatter_report(self, components: list[Dict]): + """ + Generates a formatted report for a given component based on the defined formatter. + + Parameters: + components (List[dict]): A list of dictionaries representing the components to be + processed and formatted. Each dictionary contains detailed information that adheres + to the format requirements for the specified formatter. + + Returns: + Tuple[int, dict]: A tuple where the first element represents the policy status code + and the second element is a dictionary containing formatted results information, + typically with keys 'details' and 'summary'. + + Raises: + KeyError: When a required key is missing from the provided component, causing the + formatter to fail. + ValueError: If an invalid component is passed and renders unable to process. + """ + # Get a formatter for the output results + formatter = self._get_formatter() + if formatter is None: + return PolicyStatus.ERROR.value, {} + # Format the results + data = formatter(components) + ## Save outputs if required + self.print_to_file_or_stdout(data['details'], self.output) + self.print_to_file_or_stderr(data['summary'], self.status) + # Check to see if we have policy violations + if len(components) > 0: + return PolicyStatus.POLICY_FAIL.value, data + return PolicyStatus.POLICY_SUCCESS.value, data # # End of PolicyCheck Class # \ No newline at end of file diff --git a/src/scanoss/inspection/component_summary.py b/src/scanoss/inspection/raw/component_summary.py similarity index 96% rename from src/scanoss/inspection/component_summary.py rename to src/scanoss/inspection/raw/component_summary.py index dc3d84db..6c337a82 100644 --- a/src/scanoss/inspection/component_summary.py +++ b/src/scanoss/inspection/raw/component_summary.py @@ -24,10 +24,10 @@ import json -from .inspect_base import InspectBase +from .raw_base import RawBase -class ComponentSummary(InspectBase): +class ComponentSummary(RawBase): def _get_component_summary_from_components(self, scan_components: list)-> dict: """ Get a component summary from detected components. @@ -77,7 +77,7 @@ def _get_components(self): :return: A list of processed components with license data, or `None` if `self.results` is not set. """ if self.results is None: - return None + raise ValueError(f'Error: No results found in ${self.filepath}') components: dict = {} # Extract component and license data from file and dependency results. Both helpers mutate `components` diff --git a/src/scanoss/inspection/copyleft.py b/src/scanoss/inspection/raw/copyleft.py similarity index 71% rename from src/scanoss/inspection/copyleft.py rename to src/scanoss/inspection/raw/copyleft.py index 9760e763..d778c2e5 100644 --- a/src/scanoss/inspection/copyleft.py +++ b/src/scanoss/inspection/raw/copyleft.py @@ -23,12 +23,28 @@ """ import json -from typing import Any, Dict +from dataclasses import dataclass +from typing import Any, Dict, List -from .policy_check import PolicyCheck, PolicyStatus +from ..policy_check import PolicyStatus +from .raw_base import RawBase -class Copyleft(PolicyCheck): +@dataclass +class License: + spdxid: str + copyleft: bool + url: str + source: str + +@dataclass +class Component: + purl: str + version: str + licenses: List[License] + status: str + +class Copyleft(RawBase[Component]): """ SCANOSS Copyleft class Inspects components for copyleft licenses @@ -48,7 +64,7 @@ def __init__( # noqa: PLR0913 explicit: str = None, ): """ - Initialize the Copyleft class. + Initialise the Copyleft class. :param debug: Enable debug mode :param trace: Enable trace mode (default True) @@ -61,7 +77,7 @@ def __init__( # noqa: PLR0913 :param exclude: Licenses to exclude from the analysis :param explicit: Explicitly defined licenses """ - super().__init__(debug, trace, quiet, filepath, format_type, status, output, name='Copyleft Policy') + super().__init__(debug, trace, quiet, format_type,filepath, output ,status, name='Copyleft Policy') self.license_util.init(include, exclude, explicit) self.filepath = filepath self.format = format @@ -71,7 +87,7 @@ def __init__( # noqa: PLR0913 self.exclude = exclude self.explicit = explicit - def _json(self, components: list) -> Dict[str, Any]: + def _json(self, components: list[Component]) -> Dict[str, Any]: """ Format the components with copyleft licenses as JSON. @@ -88,61 +104,67 @@ def _json(self, components: list) -> Dict[str, Any]: 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', } - def _markdown(self, components: list) -> Dict[str, Any]: + def _markdown(self, components: list[Component]) -> Dict[str, Any]: """ Format the components with copyleft licenses as Markdown. :param components: List of components with copyleft licenses :return: Dictionary with formatted Markdown details and summary """ - # A component is considered unique by its combination of PURL (Package URL) and license - component_licenses = self._group_components_by_license(components) - headers = ['Component', 'License', 'URL', 'Copyleft'] - centered_columns = [1, 4] - rows: [[]] = [] - for comp_lic_item in component_licenses: - row = [ - comp_lic_item['purl'], - comp_lic_item['spdxid'], - comp_lic_item['url'], - 'YES' if comp_lic_item['copyleft'] else 'NO', - ] - rows.append(row) - # End license loop - # End component loop - return { - 'details': f'### Copyleft licenses\n{self.generate_table(headers, rows, centered_columns)}\n', - 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', - } + return self._md_summary_generator(components, self.generate_table) - def _jira_markdown(self, components: list) -> Dict[str, Any]: + def _jira_markdown(self, components: list[Component]) -> Dict[str, Any]: """ Format the components with copyleft licenses as Markdown. :param components: List of components with copyleft licenses :return: Dictionary with formatted Markdown details and summary """ + return self._md_summary_generator(components, self.generate_jira_table) + + def _md_summary_generator(self, components: list[Component], table_generator): + """ + Generates a Markdown summary for components with a focus on copyleft licenses. + + This function processes a list of components and groups them by their licenses. + For each group, the components are mapped with their license data and a tabular representation is created. + The generated Markdown summary includes a detailed table and a summary overview. + + Parameters: + components: list[Component] + A list of Component objects to process for generating the summary. + table_generator + A callable function to generate tabular data for components. + + Returns: + dict + A dictionary containing two keys: + - 'details': A detailed Markdown representation including a table of components + and associated copyleft license data. + - 'summary': A textual summary highlighting the total number of components + with copyleft licenses. + """ # A component is considered unique by its combination of PURL (Package URL) and license component_licenses = self._group_components_by_license(components) headers = ['Component', 'License', 'URL', 'Copyleft'] centered_columns = [1, 4] - rows: [[]] = [] + rows = [] for comp_lic_item in component_licenses: - row = [ - comp_lic_item['purl'], - comp_lic_item['spdxid'], - comp_lic_item['url'], - 'YES' if comp_lic_item['copyleft'] else 'NO', - ] - rows.append(row) - # End license loop + row = [ + comp_lic_item['purl'], + comp_lic_item['spdxid'], + comp_lic_item['url'], + 'YES' if comp_lic_item['copyleft'] else 'NO', + ] + rows.append(row) + # End license loop # End component loop return { - 'details': f'{self.generate_jira_table(headers, rows, centered_columns)}', + 'details': f'### Copyleft Licenses\n{table_generator(headers, rows, centered_columns)}', 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', } - def _filter_components_with_copyleft_licenses(self, components: list) -> list: + def _get_components_with_copyleft_licenses(self, components: list) -> list[Dict]: """ Filter the components list to include only those with copyleft licenses. @@ -206,22 +228,9 @@ def run(self): if components is None: return PolicyStatus.ERROR.value, {} # Get a list of copyleft components if they exist - copyleft_components = self._filter_components_with_copyleft_licenses(components) - # Get a formatter for the output results - formatter = self._get_formatter() - if formatter is None: - return PolicyStatus.ERROR.value, {} - # Format the results - results = formatter(copyleft_components) - ## Save outputs if required - self.print_to_file_or_stdout(results['details'], self.output) - self.print_to_file_or_stderr(results['summary'], self.status) - # Check to see if we have policy violations - if len(copyleft_components) <= 0: - return PolicyStatus.FAIL.value, results - return PolicyStatus.SUCCESS.value, results - - + copyleft_components = self._get_components_with_copyleft_licenses(components) + # Format the results and save to files if required + return self._generate_formatter_report(copyleft_components) # # End of Copyleft Class # diff --git a/src/scanoss/inspection/license_summary.py b/src/scanoss/inspection/raw/license_summary.py similarity index 95% rename from src/scanoss/inspection/license_summary.py rename to src/scanoss/inspection/raw/license_summary.py index 80e06a47..4acc7dbc 100644 --- a/src/scanoss/inspection/license_summary.py +++ b/src/scanoss/inspection/raw/license_summary.py @@ -24,10 +24,10 @@ import json -from .inspect_base import InspectBase +from .raw_base import RawBase -class LicenseSummary(InspectBase): +class LicenseSummary(RawBase): """ SCANOSS LicenseSummary class Inspects results and generates comprehensive license summaries from detected components. @@ -63,7 +63,7 @@ def __init__( # noqa: PLR0913 :param exclude: Licenses to exclude from the analysis :param explicit: Explicitly defined licenses """ - super().__init__(debug, trace, quiet, filepath, output) + super().__init__(debug, trace, quiet, filepath = filepath, output=output) self.license_util.init(include, exclude, explicit) self.filepath = filepath self.output = output @@ -123,7 +123,7 @@ def _get_components(self): :return: A list of processed components with license data, or `None` if `self.results` is not set. """ if self.results is None: - return None + raise ValueError(f'Error: No results found in ${self.filepath}') components: dict = {} # Extract component and license data from file and dependency results. Both helpers mutate `components` @@ -132,6 +132,7 @@ def _get_components(self): return self._convert_components_to_list(components) def run(self): + print("Running LicenseSummary") components = self._get_components() license_summary = self._get_licenses_summary_from_components(components) self.print_to_file_or_stdout(json.dumps(license_summary, indent=2), self.output) diff --git a/src/scanoss/inspection/inspect_base.py b/src/scanoss/inspection/raw/raw_base.py similarity index 98% rename from src/scanoss/inspection/inspect_base.py rename to src/scanoss/inspection/raw/raw_base.py index 5384ee6c..0bae631e 100644 --- a/src/scanoss/inspection/inspect_base.py +++ b/src/scanoss/inspection/raw/raw_base.py @@ -26,10 +26,10 @@ import os.path from abc import abstractmethod from enum import Enum -from typing import Any, Dict +from typing import Any, Dict, TypeVar -from ..scanossbase import ScanossBase -from .utils.license_utils import LicenseUtil +from ..policy_check import PolicyCheck +from ..utils.license_utils import LicenseUtil class ComponentID(Enum): @@ -51,8 +51,8 @@ class ComponentID(Enum): # End of ComponentID Class # - -class InspectBase(ScanossBase): +T = TypeVar('T') +class RawBase(PolicyCheck[T]): """ A base class to perform inspections over scan results. @@ -68,10 +68,13 @@ def __init__( # noqa: PLR0913 debug: bool = False, trace: bool = False, quiet: bool = False, + format_type: str = None, filepath: str = None, output: str = None, + status: str = None, + name: str = None, ): - super().__init__(debug, trace, quiet) + super().__init__(debug, trace, quiet, format_type,status, name, output) self.license_util = LicenseUtil() self.filepath = filepath self.output = output diff --git a/src/scanoss/inspection/undeclared_component.py b/src/scanoss/inspection/raw/undeclared_component.py similarity index 89% rename from src/scanoss/inspection/undeclared_component.py rename to src/scanoss/inspection/raw/undeclared_component.py index 1f221f55..948dd907 100644 --- a/src/scanoss/inspection/undeclared_component.py +++ b/src/scanoss/inspection/raw/undeclared_component.py @@ -23,12 +23,27 @@ """ import json -from typing import Any, Dict +from dataclasses import dataclass +from typing import Any, Dict, List -from .policy_check import PolicyCheck, PolicyStatus +from ..policy_check import PolicyStatus +from .raw_base import RawBase -class UndeclaredComponent(PolicyCheck): +@dataclass +class License: + spdxid: str + copyleft: bool + url: str + +@dataclass +class Component: + purl: str + version: str + licenses: List[License] + status: str + +class UndeclaredComponent(RawBase[Component]): """ SCANOSS UndeclaredComponent class Inspects for undeclared components @@ -58,7 +73,7 @@ def __init__( # noqa: PLR0913 :param sbom_format: Sbom format for status output (default 'settings') """ super().__init__( - debug, trace, quiet, filepath, format_type, status, output, name='Undeclared Components Policy' + debug, trace, quiet,format_type, filepath, output, status, name='Undeclared Components Policy' ) self.filepath = filepath self.format = format @@ -66,7 +81,7 @@ def __init__( # noqa: PLR0913 self.status = status self.sbom_format = sbom_format - def _get_undeclared_component(self, components: list) -> list or None: + def _get_undeclared_components(self, components: list[Component]) -> list or None: """ Filter the components list to include only undeclared components. @@ -90,7 +105,7 @@ def _get_undeclared_component(self, components: list) -> list or None: # end component loop return undeclared_components - def _get_jira_summary(self, components: list) -> str: + def _get_jira_summary(self, components: list[Component]) -> str: """ Get a summary of the undeclared components. @@ -147,7 +162,7 @@ def _get_summary(self, components: list) -> str: return summary - def _json(self, components: list) -> Dict[str, Any]: + def _json(self, components: list[Component]) -> Dict[str, Any]: """ Format the undeclared components as JSON. @@ -164,7 +179,7 @@ def _json(self, components: list) -> Dict[str, Any]: 'summary': self._get_summary(component_licenses), } - def _markdown(self, components: list) -> Dict[str, Any]: + def _markdown(self, components: list[Component]) -> Dict[str, Any]: """ Format the undeclared components as Markdown. @@ -172,7 +187,7 @@ def _markdown(self, components: list) -> Dict[str, Any]: :return: Dictionary with formatted Markdown details and summary """ headers = ['Component', 'License'] - rows: [[]] = [] + rows = [] # TODO look at using SpdxLite license name lookup method component_licenses = self._group_components_by_license(components) for component in component_licenses: @@ -190,7 +205,7 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: :return: Dictionary with formatted Markdown details and summary """ headers = ['Component', 'License'] - rows: [[]] = [] + rows = [] # TODO look at using SpdxLite license name lookup method component_licenses = self._group_components_by_license(components) for component in component_licenses: @@ -280,24 +295,13 @@ def run(self): components = self._get_components() if components is None: return PolicyStatus.ERROR.value, {} - # Get undeclared component summary (if any) - undeclared_components = self._get_undeclared_component(components) + # Get an undeclared component summary (if any) + undeclared_components = self._get_undeclared_components(components) if undeclared_components is None: return PolicyStatus.ERROR.value, {} self.print_debug(f'Undeclared components: {undeclared_components}') - formatter = self._get_formatter() - if formatter is None: - return PolicyStatus.ERROR.value, {} - results = formatter(undeclared_components) - # Output the results - self.print_to_file_or_stdout(results['details'], self.output) - self.print_to_file_or_stderr(results['summary'], self.status) - # Determine if the filter found results or not - if len(undeclared_components) <= 0: - return PolicyStatus.FAIL.value, results - return PolicyStatus.SUCCESS.value, results - - + # Format the results and save to files if required + return self._generate_formatter_report(undeclared_components) # # End of UndeclaredComponent Class # diff --git a/src/scanoss/services/dependency_track_service.py b/src/scanoss/services/dependency_track_service.py new file mode 100644 index 00000000..e32c2452 --- /dev/null +++ b/src/scanoss/services/dependency_track_service.py @@ -0,0 +1,131 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import requests + +from ..scanossbase import ScanossBase + +HTTP_OK = 200 + +class DependencyTrackService(ScanossBase): + + def __init__( + self, + api_key: str, + url: str, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + ): + super().__init__(debug=debug, trace=trace, quiet=quiet) + if not url: + raise ValueError("Error: Dependency Track URL is required") + self.url = url.rstrip('/') + if not api_key: + raise ValueError("Error: Dependency Track API key is required") + self.api_key = api_key + + def get_project_by_name_version(self, name, version): + """ + Get project information by name and version from Dependency Track + + Args: + name: Project name to search for + version: Project version to search for + + Returns: + dict: Project data if found, None otherwise + """ + if not name or not version: + self.print_stderr('Error: Missing name or version.') + return None + # Use the project search endpoint + params = { + 'name': name, + 'version': version + } + self.print_debug(f'Searching for project by: {params}') + return self.get_dep_track_data(f'{self.url}/api/v1/project/lookup', params) + + def get_project_status(self, upload_token): + """ + Get Dependency Track project processing status. + + Queries the Dependency Track API to check if the project upload + processing is complete using the upload token. + + Returns: + dict: Project status information or None if request fails + """ + if not upload_token: + self.print_stderr('Error: Missing upload token. Cannot search for project status.') + return None + self.print_trace(f'URL: {self.url} Upload token: {upload_token}') + return self.get_dep_track_data(f'{self.url}/api/v1/event/token/{upload_token}') + + def get_project_violations(self,project_id:str): + """ + Get project violations from Dependency Track. + + Waits for project processing to complete, then retrieves all policy + violations for the specified project ID. + + Returns: + List of policy violations or None if the request fails + """ + if not project_id: + self.print_stderr('Error: Missing project id. Cannot search for project violations.') + return None + return self.get_dep_track_data(f'{self.url}/api/v1/violation/project/{project_id}') + + def get_project_by_id(self, project_id:str): + """ + Get a Dependency Track project by id. + + Queries the Dependency Track API to get a project by id + + Returns: + dict + """ + if not project_id: + self.print_stderr('Error: Missing project id. Cannot search for project.') + return None + self.print_trace(f'URL: {self.url}, UUID: {project_id}') + return self.get_dep_track_data(f'{self.url}/api/v1/project/{project_id}') + + def get_dep_track_data(self, uri, params=None): + if not uri: + self.print_stderr('Error: Missing URI. Cannot search for project.') + return None + req_headers = {'X-Api-Key': self.api_key, 'Content-Type': 'application/json'} + try: + if params: + response = requests.get(uri, headers=req_headers, params=params) + else: + response = requests.get(uri, headers=req_headers) + response.raise_for_status() # Raises an HTTPError for bad responses + return response.json() + except requests.exceptions.RequestException as e: + self.print_stderr(f"Error: Problem getting project data: {e}") + return None diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 1f2cab5d..24db1ab7 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -26,12 +26,14 @@ import os import re import unittest +from unittest.mock import Mock, patch -from scanoss.inspection.copyleft import Copyleft -from scanoss.inspection.license_summary import LicenseSummary -from scanoss.inspection.undeclared_component import UndeclaredComponent - -from src.scanoss.inspection.component_summary import ComponentSummary +from src.scanoss.inspection.policy_check import PolicyStatus +from src.scanoss.inspection.raw.component_summary import ComponentSummary +from src.scanoss.inspection.raw.copyleft import Copyleft +from src.scanoss.inspection.raw.license_summary import LicenseSummary +from src.scanoss.inspection.raw.undeclared_component import UndeclaredComponent +from src.scanoss.inspection.dependency_track.project_violation import DependencyTrackProjectViolationPolicyCheck class MyTestCase(unittest.TestCase): @@ -65,11 +67,11 @@ def test_empty_copyleft_policy(self): file_name = 'result-no-copyleft.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json') - status, results = copyleft.run() - details = json.loads(results['details']) - self.assertEqual(status, 1) + status, data = copyleft.run() + details = json.loads(data['details']) + self.assertEqual(status, PolicyStatus.POLICY_SUCCESS.value) self.assertEqual(details, {}) - self.assertEqual(results['summary'], '0 component(s) with copyleft licenses were found.\n') + self.assertEqual(data['summary'], '0 component(s) with copyleft licenses were found.\n') """ Inspect for copyleft licenses include @@ -80,16 +82,16 @@ def test_copyleft_policy_include(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', include='MIT') - status, results = copyleft.run() + status, data = copyleft.run() has_mit_license = False - details = json.loads(results['details']) + details = json.loads(data['details']) for component in details['components']: for license in component['licenses']: if license['spdxid'] == 'MIT': has_mit_license = True break - self.assertEqual(status, 0) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) self.assertEqual(has_mit_license, True) """ @@ -101,10 +103,10 @@ def test_copyleft_policy_exclude(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', exclude='GPL-2.0-only') - status, results = copyleft.run() - details = json.loads(results['details']) - self.assertEqual(details, {}) - self.assertEqual(status, 1) + status, data = copyleft.run() + results = json.loads(data['details']) + self.assertEqual(results, {}) + self.assertEqual(status, PolicyStatus.POLICY_SUCCESS.value) """ Inspect for copyleft licenses explicit @@ -115,10 +117,10 @@ def test_copyleft_policy_explicit(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='MIT') - status, results = copyleft.run() - details = json.loads(results['details']) - self.assertEqual(len(details['components']), 2) - self.assertEqual(status, 0) + status, data = copyleft.run() + results = json.loads(data['details']) + self.assertEqual(len(results['components']), 2) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) """ Inspect for copyleft licenses empty explicit licenses (should set the default ones) @@ -129,10 +131,10 @@ def test_copyleft_policy_empty_explicit(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='') - status, results = copyleft.run() - details = json.loads(results['details']) - self.assertEqual(len(details['components']), 5) - self.assertEqual(status, 0) + status, data = copyleft.run() + results = json.loads(data['details']) + self.assertEqual(len(results['components']), 5) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) """ Export copyleft licenses in Markdown @@ -143,20 +145,20 @@ def test_copyleft_policy_markdown(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='md', explicit='MIT') - status, results = copyleft.run() + status, data = copyleft.run() expected_detail_output = ( - '### Copyleft licenses \n | Component | License | URL | Copyleft |\n' + '### Copyleft Licenses \n | Component | License | URL | Copyleft |\n' ' | - | :-: | - | - |\n' ' | pkg:npm/%40electron/rebuild | MIT | https://spdx.org/licenses/MIT.html | YES |\n' '| pkg:npm/%40emotion/react | MIT | https://spdx.org/licenses/MIT.html | YES | \n' ) expected_summary_output = '2 component(s) with copyleft licenses were found.\n' self.assertEqual( - re.sub(r'\s|\\(?!`)|\\(?=`)', '', results['details']), + re.sub(r'\s|\\(?!`)|\\(?=`)', '', data['details']), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output), ) - self.assertEqual(results['summary'], expected_summary_output) - self.assertEqual(status, 0) + self.assertEqual(data['summary'], expected_summary_output) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) ## Undeclared Components Policy Tests ## @@ -164,9 +166,9 @@ def test_copyleft_policy_markdown(self): Inspect for undeclared components empty path """ - def test_copyleft_policy_empty_path(self): - copyleft = Copyleft(filepath='', format_type='json') - success, results = copyleft.run() + def test_undeclared_policy_empty_path(self): + undeclared = UndeclaredComponent(filepath='', format_type='json') + success, results = undeclared.run() self.assertTrue(success, 2) """ @@ -178,9 +180,9 @@ def test_undeclared_policy(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='json', sbom_format='legacy') - status, results = undeclared.run() - details = json.loads(results['details']) - summary = results['summary'] + status, data = undeclared.run() + results = json.loads(data['details']) + summary = data['summary'] expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json @@ -198,11 +200,11 @@ def test_undeclared_policy(self): ] }``` """ - self.assertEqual(len(details['components']), 4) + self.assertEqual(len(results['components']), 4) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) ) - self.assertEqual(status, 0) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) """ Undeclared component markdown output @@ -213,9 +215,9 @@ def test_undeclared_policy_markdown(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md', sbom_format='legacy') - status, results = undeclared.run() - details = results['details'] - summary = results['summary'] + status, data = undeclared.run() + results = data['details'] + summary = data['summary'] expected_details_output = """ ### Undeclared components | Component | License | | - | - | @@ -240,9 +242,9 @@ def test_undeclared_policy_markdown(self): ] }``` """ - self.assertEqual(status, 0) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) self.assertEqual( - re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) + re.sub(r'\s|\\(?!`)|\\(?=`)', '', results), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) ) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) @@ -257,9 +259,9 @@ def test_undeclared_policy_markdown_scanoss_summary(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md') - status, results = undeclared.run() - details = results['details'] - summary = results['summary'] + status, data = undeclared.run() + results = data['details'] + summary = data['summary'] expected_details_output = """ ### Undeclared components | Component | License | | - | - | @@ -287,9 +289,9 @@ def test_undeclared_policy_markdown_scanoss_summary(self): } } ```""" - self.assertEqual(status, 0) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) self.assertEqual( - re.sub(r'\s|\\(?!`)|\\(?=`)', '', details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) + re.sub(r'\s|\\(?!`)|\\(?=`)', '', results), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_details_output) ) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) @@ -304,9 +306,9 @@ def test_undeclared_policy_scanoss_summary(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name) - status, results = undeclared.run() - details = json.loads(results['details']) - summary = results['summary'] + status, data = undeclared.run() + results = json.loads(data['details']) + summary = data['summary'] expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file @@ -327,8 +329,8 @@ def test_undeclared_policy_scanoss_summary(self): } } ```""" - self.assertEqual(status, 0) - self.assertEqual(len(details['components']), 4) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertEqual(len(results['components']), 4) self.assertEqual( re.sub(r'\s|\\(?!`)|\\(?=`)', '', summary), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_summary_output) ) @@ -338,9 +340,9 @@ def test_undeclared_policy_jira_markdown_output(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='jira_md') - status, results = undeclared.run() - details = results['details'] - summary = results['summary'] + status, data = undeclared.run() + details = data['details'] + summary = data['summary'] expected_details_output = """|*Component*|*License*| |pkg:github/scanoss/jenkins-pipeline-example|unknown| |pkg:github/scanoss/scanner.c|GPL-2.0-only| @@ -366,7 +368,7 @@ def test_undeclared_policy_jira_markdown_output(self): } {code} """ - self.assertEqual(status, 0) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) self.assertEqual(expected_details_output, details) self.assertEqual(summary, expected_summary_output) @@ -375,15 +377,15 @@ def test_copyleft_policy_jira_markdown_output(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='jira_md') - status, results = copyleft.run() - details = results['details'] - expected_details_output = """|*Component*|*License*|*URL*|*Copyleft*| + status, data = copyleft.run() + results = data['details'] + expected_details_output = """### Copyleft Licenses\n|*Component*|*License*|*URL*|*Copyleft*| |pkg:github/scanoss/scanner.c|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| |pkg:github/scanoss/engine|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| |pkg:github/scanoss/wfp|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| """ - self.assertEqual(status, 0) - self.assertEqual(expected_details_output, details) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertEqual(expected_details_output, results) def test_inspect_license_summary(self): script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -432,5 +434,177 @@ def test_inspect_component_summary_empty_result(self): self.assertEqual(component_summary['totalFilesUndeclared'], 0) self.assertEqual(component_summary['totalFilesDeclared'], 0) + ## Dependency Track Project Violation Policy Tests ## + + @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + def test_dependency_track_project_violation_json_formatter(self, mock_service): + mock_service.return_value = Mock() + project_violation = DependencyTrackProjectViolationPolicyCheck( + format_type='json', + api_key='test_key', + url='http://localhost', + project_id='test_project' + ) + test_violations = [ + { + 'uuid': 'violation-1', + 'type': 'SECURITY', + 'timestamp': 1640995200000, + 'component': { + 'name': 'test-component', + 'version': '1.0.0', + 'purl': 'pkg:npm/test-component@1.0.0' + }, + 'policyCondition': { + 'policy': { + 'name': 'Security Policy', + 'violationState': 'FAIL' + } + } + } + ] + result = project_violation._json(test_violations) + self.assertIn('details', result) + self.assertIn('summary', result) + self.assertEqual(result['summary'], '1 policy violations were found.\n') + details = json.loads(result['details']) + self.assertEqual(len(details), 1) + self.assertEqual(details[0]['type'], 'SECURITY') + + @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + def test_dependency_track_project_violation_markdown_formatter(self, mock_service): + mock_service.return_value = Mock() + project_violation = DependencyTrackProjectViolationPolicyCheck( + format_type='md', + api_key='test_key', + url='http://localhost', + project_id='test_project' + ) + test_violations = [ + { + 'uuid': 'violation-1', + 'type': 'SECURITY', + 'timestamp': 1640995200000, + 'component': { + 'name': 'test-component', + 'version': '1.0.0', + 'purl': 'pkg:npm/test-component@1.0.0' + }, + 'policyCondition': { + 'policy': { + 'name': 'Security Policy', + 'violationState': 'FAIL' + } + } + } + ] + result = project_violation._markdown(test_violations) + self.assertIn('details', result) + self.assertIn('summary', result) + self.assertEqual(result['summary'], '1 policy violations were found.\n') + self.assertIn('State', result['details']) + self.assertIn('Risk Type', result['details']) + self.assertIn('Policy Name', result['details']) + self.assertIn('Component', result['details']) + self.assertIn('Date', result['details']) + + @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + def test_dependency_track_project_violation_sort_violations(self, mock_service): + mock_service.return_value = Mock() + project_violation = DependencyTrackProjectViolationPolicyCheck( + api_key='test_key', + url='http://localhost', + project_id='test_project' + ) + test_violations = [ + {'type': 'LICENSE', 'uuid': 'license-violation'}, + {'type': 'SECURITY', 'uuid': 'security-violation'}, + {'type': 'OTHER', 'uuid': 'other-violation'}, + {'type': 'SECURITY', 'uuid': 'security-violation-2'} + ] + sorted_violations = project_violation._sort_project_violations(test_violations) + self.assertEqual(sorted_violations[0]['type'], 'SECURITY') + self.assertEqual(sorted_violations[1]['type'], 'SECURITY') + self.assertEqual(sorted_violations[2]['type'], 'LICENSE') + self.assertEqual(sorted_violations[3]['type'], 'OTHER') + + @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + def test_dependency_track_project_violation_empty_violations(self, mock_service): + mock_service.return_value = Mock() + project_violation = DependencyTrackProjectViolationPolicyCheck( + format_type='json', + api_key='test_key', + url='http://localhost', + project_id='test_project' + ) + empty_violations = [] + result = project_violation._json(empty_violations) + self.assertEqual(result['summary'], '0 policy violations were found.\n') + details = json.loads(result['details']) + self.assertEqual(len(details), 0) + + @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + def test_dependency_track_project_violation_markdown_empty(self, mock_service): + mock_service.return_value = Mock() + project_violation = DependencyTrackProjectViolationPolicyCheck( + format_type='md', + api_key='test_key', + url='http://localhost', + project_id='test_project' + ) + empty_violations = [] + result = project_violation._markdown(empty_violations) + self.assertEqual(result['summary'], '0 policy violations were found.\n') + self.assertIn('State', result['details']) + self.assertIn('Risk Type', result['details']) + + @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + def test_dependency_track_project_violation_multiple_types(self, mock_service): + mock_service.return_value = Mock() + project_violation = DependencyTrackProjectViolationPolicyCheck( + format_type='json', + api_key='test_key', + url='http://localhost', + project_id='test_project' + ) + test_violations = [ + { + 'uuid': 'violation-1', + 'type': 'SECURITY', + 'timestamp': 1640995200000, + 'component': { + 'name': 'vulnerable-component', + 'version': '1.0.0', + 'purl': 'pkg:npm/vulnerable-component@1.0.0' + }, + 'policyCondition': { + 'policy': { + 'name': 'Security Policy', + 'violationState': 'FAIL' + } + } + }, + { + 'uuid': 'violation-2', + 'type': 'LICENSE', + 'timestamp': 1640995300000, + 'component': { + 'name': 'license-component', + 'version': '2.0.0', + 'purl': 'pkg:npm/license-component@2.0.0' + }, + 'policyCondition': { + 'policy': { + 'name': 'License Policy', + 'violationState': 'WARN' + } + } + } + ] + result = project_violation._json(test_violations) + self.assertEqual(result['summary'], '2 policy violations were found.\n') + details = json.loads(result['details']) + self.assertEqual(len(details), 2) + if __name__ == '__main__': unittest.main() From 16b743f8a48d8e45cec4c70f42f7d25f4df2fe3d Mon Sep 17 00:00:00 2001 From: Alex-1089 Date: Fri, 8 Aug 2025 15:20:03 +0100 Subject: [PATCH 384/489] Fixing DT md formatting bug (#142) --- CHANGELOG.md | 6 +++++- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 3 +-- .../inspection/dependency_track/project_violation.py | 10 ++++++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01c4812..1a25f57f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.31.1] - 2025-08-08 +### Fixed +- Fixed purl formatting bug in dependency track output + ## [1.31.0] - 2025-08-08 ### Added - Add `inspect dependency-track project-violations` subcommand to retrieve Dependency Track project violations in Markdown and JSON formats @@ -628,4 +632,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.29.0]: https://github.com/scanoss/scanoss.py/compare/v1.28.2...v1.29.0 [1.30.0]: https://github.com/scanoss/scanoss.py/compare/v1.29.0...v1.30.0 [1.31.0]: https://github.com/scanoss/scanoss.py/compare/v1.30.0...v1.31.0 - +[1.31.1]: https://github.com/scanoss/scanoss.py/compare/v1.31.0...v1.31.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 1868ffa4..83a076db 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.31.0' +__version__ = '1.31.1' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 057eedc3..abbfdc3e 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1800,9 +1800,8 @@ def inspect_dep_track_project_violations(parser, args): project_version=args.project_version, # DT project version timeout=args.timeout, ) - # Execute inspection and exit with appropriate status code - status, _ = dt_proj_violations.run() #TODO remove datastructure from return + status = dt_proj_violations.run() sys.exit(status) except Exception as e: print_stderr(e) diff --git a/src/scanoss/inspection/dependency_track/project_violation.py b/src/scanoss/inspection/dependency_track/project_violation.py index 0218d599..0216ab87 100644 --- a/src/scanoss/inspection/dependency_track/project_violation.py +++ b/src/scanoss/inspection/dependency_track/project_violation.py @@ -384,15 +384,21 @@ def _md_summary_generator(self, project_violations: list[PolicyViolationDict], t timestamp_seconds = timestamp / MILLISECONDS_TO_SECONDS formatted_date = datetime.fromtimestamp(timestamp_seconds).strftime("%d %b %Y at %H:%M:%S") + purl = project_violation["component"]["purl"] + version = project_violation["component"]["version"] + # If PURL doesn't contain version but version is available, append it + component_display = purl + if version and '@' not in purl: + component_display = f'{purl}@{version}' row = [ project_violation['policyCondition']["policy"]["violationState"], project_violation['type'], project_violation['policyCondition']["policy"]["name"], - f'{project_violation["component"]["purl"]}@{project_violation["component"]["version"]}', + component_display, formatted_date, ] rows.append(row) - + # End for loop return { "details": f'### Dependency Track Project Violations\n{table_generator(headers, rows, c_cols)}\n', "summary": f'{len(project_violations)} policy violations were found.\n', From fdc02afa768db6f5bc51909b3f4f6bfc6b71b6d3 Mon Sep 17 00:00:00 2001 From: Alex-1089 Date: Tue, 12 Aug 2025 11:39:40 +0100 Subject: [PATCH 385/489] Removed unnecessary print --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- src/scanoss/inspection/raw/license_summary.py | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a25f57f..6c6af6ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.31.2] - 2025-08-12 +### Fixed +- Removed an unnecessary print statement from the policy checker + ## [1.31.1] - 2025-08-08 ### Fixed - Fixed purl formatting bug in dependency track output @@ -633,3 +637,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.30.0]: https://github.com/scanoss/scanoss.py/compare/v1.29.0...v1.30.0 [1.31.0]: https://github.com/scanoss/scanoss.py/compare/v1.30.0...v1.31.0 [1.31.1]: https://github.com/scanoss/scanoss.py/compare/v1.31.0...v1.31.1 +[1.31.2]: https://github.com/scanoss/scanoss.py/compare/v1.31.1...v1.31.2 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 83a076db..a16668c3 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.31.1' +__version__ = '1.31.2' diff --git a/src/scanoss/inspection/raw/license_summary.py b/src/scanoss/inspection/raw/license_summary.py index 4acc7dbc..bd85c56d 100644 --- a/src/scanoss/inspection/raw/license_summary.py +++ b/src/scanoss/inspection/raw/license_summary.py @@ -132,7 +132,6 @@ def _get_components(self): return self._convert_components_to_list(components) def run(self): - print("Running LicenseSummary") components = self._get_components() license_summary = self._get_licenses_summary_from_components(components) self.print_to_file_or_stdout(json.dumps(license_summary, indent=2), self.output) From 3900859836c18ec68261bbfeb4c2a9e89f5ecd33 Mon Sep 17 00:00:00 2001 From: Alex-1089 Date: Tue, 19 Aug 2025 18:12:02 +0100 Subject: [PATCH 386/489] Added handling for empty results files (#145) * Added handling for empty results files * Fixed Docker and linter issues --- CHANGELOG.md | 5 ++++ Dockerfile | 2 +- src/scanoss/__init__.py | 2 +- src/scanoss/csvoutput.py | 20 ++++++++----- src/scanoss/cyclonedx.py | 11 +++++-- src/scanoss/export/dependency_track.py | 7 ++++- .../dependency_track/project_violation.py | 30 ++++++++++++++++--- .../services/dependency_track_service.py | 1 + src/scanoss/spdxlite.py | 9 ++++-- 9 files changed, 68 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6af6ab..a422aa7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.31.3] - 2025-08-19 +### Fixed +- Added handling for empty results files + ## [1.31.2] - 2025-08-12 ### Fixed - Removed an unnecessary print statement from the policy checker @@ -638,3 +642,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.31.0]: https://github.com/scanoss/scanoss.py/compare/v1.30.0...v1.31.0 [1.31.1]: https://github.com/scanoss/scanoss.py/compare/v1.31.0...v1.31.1 [1.31.2]: https://github.com/scanoss/scanoss.py/compare/v1.31.1...v1.31.2 +[1.31.2]: https://github.com/scanoss/scanoss.py/compare/v1.31.2...v1.31.3 diff --git a/Dockerfile b/Dockerfile index 14d18c2c..f1fb4648 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM python:3.10-slim AS base +FROM --platform=$BUILDPLATFORM python:3.10-slim-bookworm AS base LABEL maintainer="SCANOSS " LABEL org.opencontainers.image.source=https://github.com/scanoss/scanoss.py diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index a16668c3..40a8be75 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.31.2' +__version__ = '1.31.3' diff --git a/src/scanoss/csvoutput.py b/src/scanoss/csvoutput.py index 5b36284c..5a7b1974 100644 --- a/src/scanoss/csvoutput.py +++ b/src/scanoss/csvoutput.py @@ -21,11 +21,10 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - +import csv import json import os.path import sys -import csv from .scanossbase import ScanossBase @@ -44,16 +43,20 @@ def __init__(self, debug: bool = False, output_file: str = None): self.output_file = output_file self.debug = debug - def parse(self, data: json): + # TODO Refactor (fails linter) + def parse(self, data: json): #noqa PLR0912, PLR0915 """ Parse the given input (raw/plain) JSON string and return CSV summary :param data: json - JSON object :return: CSV dictionary """ - if not data: + if data is None: self.print_stderr('ERROR: No JSON data provided to parse.') return None - self.print_debug(f'Processing raw results into CSV format...') + if len(data) == 0: + self.print_msg('Warning: Empty scan results provided. Returning empty CSV list.') + return [] + self.print_debug('Processing raw results into CSV format...') csv_dict = [] row_id = 1 for f in data: @@ -92,7 +95,8 @@ def parse(self, data: json): detected['licenses'] = '' else: detected['licenses'] = ';'.join(dc) - # inventory_id,path,usage,detected_component,detected_license,detected_version,detected_latest,purl + # inventory_id,path,usage,detected_component,detected_license, + # detected_version,detected_latest,purl csv_dict.append( { 'inventory_id': row_id, @@ -183,9 +187,11 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: :return: True if successful, False otherwise """ csv_data = self.parse(data) - if not csv_data: + if csv_data is None: self.print_stderr('ERROR: No CSV data returned for the JSON string provided.') return False + if len(csv_data) == 0: + self.print_msg('Warning: Empty scan results - generating CSV with headers only.') # Header row/column details fields = [ 'inventory_id', diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index d6ad389a..f3685a97 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -57,9 +57,12 @@ def parse(self, data: dict): # noqa: PLR0912, PLR0915 :param data: dict - JSON object :return: CycloneDX dictionary, and vulnerability dictionary """ - if not data: + if data is None: self.print_stderr('ERROR: No JSON data provided to parse.') return None, None + if len(data) == 0: + self.print_msg('Warning: Empty scan results provided. Returning empty component dictionary.') + return {}, {} self.print_debug('Processing raw results into CycloneDX format...') cdx = {} vdx = {} @@ -186,9 +189,11 @@ def produce_from_json(self, data: dict, output_file: str = None) -> tuple[bool, json: The CycloneDX output """ cdx, vdx = self.parse(data) - if not cdx: + if cdx is None: self.print_stderr('ERROR: No CycloneDX data returned for the JSON string provided.') - return False, None + return False, {} + if len(cdx) == 0: + self.print_msg('Warning: Empty scan results - generating minimal CycloneDX SBOM with no components.') self._spdx.load_license_data() # Load SPDX license name data for later reference # # Using CDX version 1.4: https://cyclonedx.org/docs/1.4/json/ diff --git a/src/scanoss/export/dependency_track.py b/src/scanoss/export/dependency_track.py index 38dccd4d..f3969f40 100644 --- a/src/scanoss/export/dependency_track.py +++ b/src/scanoss/export/dependency_track.py @@ -118,7 +118,12 @@ def _encode_sbom(self, sbom_content: dict) -> str: Base64 encoded string """ if not sbom_content: - self.print_stderr('Warning: Empty SBOM content') + self.print_stderr('Warning: Empty SBOM content provided') + return '' + # Check if SBOM has no components (empty scan results) + components = sbom_content.get('components', []) + if len(components) == 0: + self.print_msg('Notice: SBOM contains no components (empty scan results)') json_str = json.dumps(sbom_content, separators=(',', ':')) encoded = base64.b64encode(json_str.encode('utf-8')).decode('utf-8') return encoded diff --git a/src/scanoss/inspection/dependency_track/project_violation.py b/src/scanoss/inspection/dependency_track/project_violation.py index 0216ab87..a5876682 100644 --- a/src/scanoss/inspection/dependency_track/project_violation.py +++ b/src/scanoss/inspection/dependency_track/project_violation.py @@ -230,16 +230,34 @@ def is_project_updated(self, dt_project: Dict[str, Any]) -> bool: if not dt_project: self.print_stderr('Warning: No project details supplied. Returning False.') return False - last_import = dt_project.get('lastBomImport', 0) - last_vulnerability_analysis = dt_project.get('lastVulnerabilityAnalysis', 0) + + # Safely extract and normalise timestamp values to numeric types + def _safe_timestamp(field, value=None, default=0) -> float: + """Convert timestamp value to float, handling string/numeric types safely.""" + if value is None: + return float(default) + try: + return float(value) + except (ValueError, TypeError): + self.print_stderr(f'Warning: Invalid timestamp for {field}, value: {value}, using default: {default}') + return float(default) + + last_import = _safe_timestamp('lastBomImport', dt_project.get('lastBomImport'), 0) + last_vulnerability_analysis = _safe_timestamp('lastVulnerabilityAnalysis', + dt_project.get('lastVulnerabilityAnalysis'), 0 + ) metrics = dt_project.get('metrics', {}) - last_occurrence = metrics.get('lastOccurrence', 0) if isinstance(metrics, dict) else 0 + last_occurrence = _safe_timestamp('lastOccurrence', + metrics.get('lastOccurrence', 0) + if isinstance(metrics, dict) else 0, 0 + ) if self.debug: self.print_msg(f'last_import: {last_import}') self.print_msg(f'last_vulnerability_analysis: {last_vulnerability_analysis}') self.print_msg(f'last_occurrence: {last_occurrence}') self.print_msg(f'last_vulnerability_analysis is updated: {last_vulnerability_analysis >= last_import}') self.print_msg(f'last_occurrence is updated: {last_occurrence >= last_import}') + # If all timestamps are zero, this indicates no processing has occurred if last_vulnerability_analysis == 0 or last_occurrence == 0 or last_import == 0: self.print_stderr(f'Warning: Some project data appears to be unset. Returning False: {dt_project}') return False @@ -434,12 +452,16 @@ def run(self) -> int: return PolicyStatus.ERROR.value # Get project violations from Dependency Track dt_project_violations = self.dep_track_service.get_project_violations(self.project_id) + # Handle case where service returns None (API error) vs empty list (no violations) + if dt_project_violations is None: + self.print_stderr('Error: Failed to retrieve project violations from Dependency Track') + return PolicyStatus.ERROR.value # Sort violations by priority and format output formatter = self._get_formatter() if formatter is None: self.print_stderr('Error: Invalid format specified.') return PolicyStatus.ERROR.value - # Format and output data + # Format and output data - handle empty results gracefully data = formatter(self._sort_project_violations(dt_project_violations)) self.print_to_file_or_stdout(data['details'], self.output) self.print_to_file_or_stderr(data['summary'], self.status) diff --git a/src/scanoss/services/dependency_track_service.py b/src/scanoss/services/dependency_track_service.py index e32c2452..7a367a0f 100644 --- a/src/scanoss/services/dependency_track_service.py +++ b/src/scanoss/services/dependency_track_service.py @@ -97,6 +97,7 @@ def get_project_violations(self,project_id:str): if not project_id: self.print_stderr('Error: Missing project id. Cannot search for project violations.') return None + # Return the result as-is - None indicates API failure, empty list means no violations return self.get_dep_track_data(f'{self.url}/api/v1/violation/project/{project_id}') def get_project_by_id(self, project_id:str): diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index fe864eba..7313b271 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -71,9 +71,12 @@ def parse(self, data: json): :param data: json - JSON object :return: summary dictionary """ - if not data: + if data is None: self.print_stderr('ERROR: No JSON data provided to parse.') return None + if len(data) == 0: + self.print_debug('Warning: Empty scan results provided. Returning empty summary.') + return {} self.print_debug('Processing raw results into summary format...') return self._process_files(data) @@ -277,9 +280,11 @@ def produce_from_json(self, data: json, output_file: str = None) -> bool: :return: True if successful, False otherwise """ raw_data = self.parse(data) - if not raw_data: + if raw_data is None: self.print_stderr('ERROR: No SPDX data returned for the JSON string provided.') return False + if len(raw_data) == 0: + self.print_debug('Warning: Empty scan results - generating minimal SPDX Lite document with no packages.') self.load_license_data() spdx_document = self._create_base_document(raw_data) From e1320cc2c16b456358bcb67ed02b685570d69891 Mon Sep 17 00:00:00 2001 From: Alex-1089 Date: Wed, 20 Aug 2025 17:07:42 +0100 Subject: [PATCH 387/489] Added catch for empty results file in dependency track (#146) * Added catch for empty results file in dependency track --- CHANGELOG.md | 7 ++++++- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 3 ++- .../inspection/dependency_track/project_violation.py | 8 +++++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a422aa7f..50d7f25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.31.4] - 2025-08-20 +### Added +- Added support for empty dependency track project policy checks + ## [1.31.3] - 2025-08-19 ### Fixed - Added handling for empty results files @@ -642,4 +646,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.31.0]: https://github.com/scanoss/scanoss.py/compare/v1.30.0...v1.31.0 [1.31.1]: https://github.com/scanoss/scanoss.py/compare/v1.31.0...v1.31.1 [1.31.2]: https://github.com/scanoss/scanoss.py/compare/v1.31.1...v1.31.2 -[1.31.2]: https://github.com/scanoss/scanoss.py/compare/v1.31.2...v1.31.3 +[1.31.3]: https://github.com/scanoss/scanoss.py/compare/v1.31.2...v1.31.3 +[1.31.4]: https://github.com/scanoss/scanoss.py/compare/v1.31.3...v1.31.4 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 40a8be75..9287075b 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.31.3' +__version__ = '1.31.4' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index abbfdc3e..7e447522 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -803,7 +803,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_inspect_dt_project_violation.add_argument( '--timeout', '-M', required=False, - default='300', + default=300, + type=float, help='Timeout (in seconds) for API communication (optional - default 300 sec)' ) diff --git a/src/scanoss/inspection/dependency_track/project_violation.py b/src/scanoss/inspection/dependency_track/project_violation.py index a5876682..9e2d5ed2 100644 --- a/src/scanoss/inspection/dependency_track/project_violation.py +++ b/src/scanoss/inspection/dependency_track/project_violation.py @@ -31,7 +31,7 @@ # Constants PROCESSING_RETRY_DELAY = 5 # seconds -DEFAULT_TIME_OUT = 300 +DEFAULT_TIME_OUT = 300.0 MILLISECONDS_TO_SECONDS = 1000 @@ -257,6 +257,12 @@ def _safe_timestamp(field, value=None, default=0) -> float: self.print_msg(f'last_occurrence: {last_occurrence}') self.print_msg(f'last_vulnerability_analysis is updated: {last_vulnerability_analysis >= last_import}') self.print_msg(f'last_occurrence is updated: {last_occurrence >= last_import}') + # Catches case where vulnerability analysis is skipped for empty SBOMs + if 0 < last_import <= last_occurrence: + component_count = metrics.get('components', 0) if isinstance(metrics, dict) else 0 + if component_count < 1: + self.print_msg('Notice: Empty SBOM detected. Assuming no violations.') + return True # If all timestamps are zero, this indicates no processing has occurred if last_vulnerability_analysis == 0 or last_occurrence == 0 or last_import == 0: self.print_stderr(f'Warning: Some project data appears to be unset. Returning False: {dt_project}') From 693425b0c6e2259dcdbdf1eb9dc00e2f23b44607 Mon Sep 17 00:00:00 2001 From: Alex-1089 Date: Wed, 27 Aug 2025 10:30:08 +0100 Subject: [PATCH 388/489] dt-url-markdown (#148) * Updated version to 1.31.5. Added DT url to markdown summary. * updated protobuf api definitions * add missing grpc dependency * increase arg limit to 6 * update declared components --------- Co-authored-by: eeisegn --- CHANGELOG.md | 8 + pyproject.toml | 2 +- requirements.txt | 2 +- scanoss.json | 26 +- setup.cfg | 2 +- src/scanoss/__init__.py | 2 +- .../api/common/v2/scanoss_common_pb2.py | 69 ++-- .../api/common/v2/scanoss_common_pb2_grpc.py | 20 ++ .../components/v2/scanoss_components_pb2.py | 97 +++--- .../v2/scanoss_components_pb2_grpc.py | 93 +++++- .../v2/scanoss_cryptography_pb2.py | 105 +++--- .../v2/scanoss_cryptography_pb2_grpc.py | 129 ++++++-- .../v2/scanoss_dependencies_pb2.py | 85 ++--- .../v2/scanoss_dependencies_pb2_grpc.py | 75 ++++- .../v2/scanoss_geoprovenance_pb2.py | 73 +++-- .../v2/scanoss_geoprovenance_pb2_grpc.py | 75 ++++- src/scanoss/api/licenses/__init__.py | 23 ++ src/scanoss/api/licenses/v2/__init__.py | 23 ++ .../api/licenses/v2/scanoss_licenses_pb2.py | 84 +++++ .../licenses/v2/scanoss_licenses_pb2_grpc.py | 302 ++++++++++++++++++ .../api/scanning/v2/scanoss_scanning_pb2.py | 49 +-- .../scanning/v2/scanoss_scanning_pb2_grpc.py | 57 +++- .../api/semgrep/v2/scanoss_semgrep_pb2.py | 57 ++-- .../semgrep/v2/scanoss_semgrep_pb2_grpc.py | 57 +++- .../v2/scanoss_vulnerabilities_pb2.py | 109 +++++-- .../v2/scanoss_vulnerabilities_pb2_grpc.py | 300 +++++++++++++++-- src/scanoss/cli.py | 4 +- .../dependency_track/project_violation.py | 17 +- .../services/dependency_track_service.py | 2 +- src/scanoss/threadeddependencies.py | 37 +-- 30 files changed, 1581 insertions(+), 403 deletions(-) create mode 100644 src/scanoss/api/licenses/__init__.py create mode 100644 src/scanoss/api/licenses/v2/__init__.py create mode 100644 src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py create mode 100644 src/scanoss/api/licenses/v2/scanoss_licenses_pb2_grpc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 50d7f25d..62180f9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.31.5] - 2025-08-27 +### Added +- Added jira markdown option for DT +- Added Dependency Track project link to markdown summary +- Updated protobuf client definitions +- Added date field to `scanoss-py comp versions` response + ## [1.31.4] - 2025-08-20 ### Added - Added support for empty dependency track project policy checks @@ -648,3 +655,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.31.2]: https://github.com/scanoss/scanoss.py/compare/v1.31.1...v1.31.2 [1.31.3]: https://github.com/scanoss/scanoss.py/compare/v1.31.2...v1.31.3 [1.31.4]: https://github.com/scanoss/scanoss.py/compare/v1.31.3...v1.31.4 +[1.31.5]: https://github.com/scanoss/scanoss.py/compare/v1.31.4...v1.31.5 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 0925b1e2..a3163499 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,4 +24,4 @@ line-ending = "auto" known-first-party = ["scanoss"] [tool.ruff.lint.pylint] -max-args = 5 +max-args = 6 diff --git a/requirements.txt b/requirements.txt index 4add4629..1d3fc07c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,5 @@ packageurl-python pathspec jsonschema crc - +protoc-gen-openapiv2 cyclonedx-python-lib[validation] \ No newline at end of file diff --git a/scanoss.json b/scanoss.json index 37724c38..813b121a 100644 --- a/scanoss.json +++ b/scanoss.json @@ -1,7 +1,13 @@ { "settings": { "skip": { - "patterns": {}, + "patterns": { + "scanning": [ + "src/protoc_gen_swagger/", + "src/scanoss/api/", + "docs/make.bat" + ] + }, "sizes": {} } }, @@ -10,24 +16,6 @@ { "purl": "pkg:github/scanoss/scanoss.py" } - ], - "remove": [ - { - "path": "docs/make.bat", - "purl": "pkg:github/twilight-logic/ar488" - }, - { - "path": "src/protoc_gen_swagger/options/annotations_pb2_grpc.py", - "purl": "pkg:pypi/bauplan" - }, - { - "path": "src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py", - "purl": "pkg:pypi/bauplan" - }, - { - "path": "src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py", - "purl": "pkg:pypi/bauplan" - } ] } } \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 36e0d74b..cb37a860 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,9 +39,9 @@ install_requires = pathspec jsonschema crc + protoc-gen-openapiv2 cyclonedx-python-lib[validation] - [options.extras_require] fast_winnowing = scanoss_winnowing>=0.5.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 9287075b..d37886ff 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.31.4' +__version__ = '1.31.5' diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index cbcd2e5b..a82eb997 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -1,38 +1,63 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/common/v2/scanoss-common.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/common/v2/scanoss-common.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 +from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\")\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\x1a.protoc-gen-openapiv2/options/annotations.proto\x1a\x1fgoogle/api/field_behavior.proto\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"k\n\x10\x43omponentRequest\x12\x11\n\x04purl\x18\x01 \x01(\tB\x03\xe0\x41\x02\x12\x13\n\x0brequirement\x18\x02 \x01(\t:/\x92\x41,2*{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"}\"\xc8\x01\n\x11\x43omponentsRequest\x12@\n\ncomponents\x18\x01 \x03(\x0b\x32\'.scanoss.api.common.v2.ComponentRequestB\x03\xe0\x41\x02:q\x92\x41n2l{\"components\":[{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"},{\"purl\":\"pkg:github/scanoss/scanoss.py@v1.30.0\"}]}\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\")\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' - _STATUSCODE._serialized_start=379 - _STATUSCODE._serialized_end=475 - _STATUSRESPONSE._serialized_start=69 - _STATUSRESPONSE._serialized_end=153 - _ECHOREQUEST._serialized_start=155 - _ECHOREQUEST._serialized_end=185 - _ECHORESPONSE._serialized_start=187 - _ECHORESPONSE._serialized_end=218 - _PURLREQUEST._serialized_start=220 - _PURLREQUEST._serialized_end=334 - _PURLREQUEST_PURLS._serialized_start=292 - _PURLREQUEST_PURLS._serialized_end=334 - _PURL._serialized_start=336 - _PURL._serialized_end=377 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.common.v2.scanoss_common_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2' + _globals['_COMPONENTREQUEST'].fields_by_name['purl']._loaded_options = None + _globals['_COMPONENTREQUEST'].fields_by_name['purl']._serialized_options = b'\340A\002' + _globals['_COMPONENTREQUEST']._loaded_options = None + _globals['_COMPONENTREQUEST']._serialized_options = b'\222A,2*{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"}' + _globals['_COMPONENTSREQUEST'].fields_by_name['components']._loaded_options = None + _globals['_COMPONENTSREQUEST'].fields_by_name['components']._serialized_options = b'\340A\002' + _globals['_COMPONENTSREQUEST']._loaded_options = None + _globals['_COMPONENTSREQUEST']._serialized_options = b'\222An2l{\"components\":[{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"},{\"purl\":\"pkg:github/scanoss/scanoss.py@v1.30.0\"}]}' + _globals['_STATUSCODE']._serialized_start=772 + _globals['_STATUSCODE']._serialized_end=868 + _globals['_STATUSRESPONSE']._serialized_start=150 + _globals['_STATUSRESPONSE']._serialized_end=234 + _globals['_ECHOREQUEST']._serialized_start=236 + _globals['_ECHOREQUEST']._serialized_end=266 + _globals['_ECHORESPONSE']._serialized_start=268 + _globals['_ECHORESPONSE']._serialized_end=299 + _globals['_COMPONENTREQUEST']._serialized_start=301 + _globals['_COMPONENTREQUEST']._serialized_end=408 + _globals['_COMPONENTSREQUEST']._serialized_start=411 + _globals['_COMPONENTSREQUEST']._serialized_end=611 + _globals['_PURLREQUEST']._serialized_start=613 + _globals['_PURLREQUEST']._serialized_end=727 + _globals['_PURLREQUEST_PURLS']._serialized_start=685 + _globals['_PURLREQUEST_PURLS']._serialized_end=727 + _globals['_PURL']._serialized_start=729 + _globals['_PURL']._serialized_end=770 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index 2daafffe..b5c3c03c 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -1,4 +1,24 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings + +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/common/v2/scanoss_common_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index cf1290a9..3820ed92 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/components/v2/scanoss-components.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/components/v2/scanoss-components.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,49 +24,49 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 - +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05\"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xd6\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\x64\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xd4\x04\n\nComponents\x12s\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/api/v2/components/echo:\x01*\x12\x95\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/components/search:\x01*\x12\x9d\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"&\x82\xd3\xe4\x93\x02 \"\x1b/api/v2/components/versions:\x01*\x12\x98\x01\n\x16GetComponentStatistics\x12\".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse\"(\x82\xd3\xe4\x93\x02\"\"\x1d/api/v2/components/statistics:\x01*B\x94\x02Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd9\x01\x12s\n\x1aSCANOSS Components Service\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05\"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xe4\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1ar\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x12\x0c\n\x04\x64\x61te\x18\x05 \x01(\t\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xcb\x04\n\nComponents\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\x12\x17/api/v2/components/echo\x12\x92\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"!\x82\xd3\xe4\x93\x02\x1b\x12\x19/api/v2/components/search\x12\x9a\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"#\x82\xd3\xe4\x93\x02\x1d\x12\x1b/api/v2/components/versions\x12\x98\x01\n\x16GetComponentStatistics\x12\".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse\"(\x82\xd3\xe4\x93\x02\"\"\x1d/api/v2/components/statistics:\x01*B\x9a\x03Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xdf\x02\x12\x9d\x01\n\x1aSCANOSS Components Service\x12(Provides component intelligence services\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x02\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07Z8\n6\n\x07\x61pi_key\x12+\x08\x02\x12\x1a\x41PI key for authentication\x1a\tx-api-key \x02\x62\r\n\x0b\n\x07\x61pi_key\x12\x00\x62\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\331\001\022s\n\032SCANOSS Components Service\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _COMPONENTS.methods_by_name['Echo']._options = None - _COMPONENTS.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\034\"\027/api/v2/components/echo:\001*' - _COMPONENTS.methods_by_name['SearchComponents']._options = None - _COMPONENTS.methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/components/search:\001*' - _COMPONENTS.methods_by_name['GetComponentVersions']._options = None - _COMPONENTS.methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002 \"\033/api/v2/components/versions:\001*' - _COMPONENTS.methods_by_name['GetComponentStatistics']._options = None - _COMPONENTS.methods_by_name['GetComponentStatistics']._serialized_options = b'\202\323\344\223\002\"\"\035/api/v2/components/statistics:\001*' - _COMPSEARCHREQUEST._serialized_start=201 - _COMPSEARCHREQUEST._serialized_end=319 - _COMPSTATISTIC._serialized_start=322 - _COMPSTATISTIC._serialized_end=524 - _COMPSTATISTIC_LANGUAGE._serialized_start=485 - _COMPSTATISTIC_LANGUAGE._serialized_end=524 - _COMPSTATISTICRESPONSE._serialized_start=527 - _COMPSTATISTICRESPONSE._serialized_end=778 - _COMPSTATISTICRESPONSE_PURLS._serialized_start=678 - _COMPSTATISTICRESPONSE_PURLS._serialized_end=778 - _COMPSEARCHRESPONSE._serialized_start=781 - _COMPSEARCHRESPONSE._serialized_end=992 - _COMPSEARCHRESPONSE_COMPONENT._serialized_start=935 - _COMPSEARCHRESPONSE_COMPONENT._serialized_end=992 - _COMPVERSIONREQUEST._serialized_start=994 - _COMPVERSIONREQUEST._serialized_end=1043 - _COMPVERSIONRESPONSE._serialized_start=1046 - _COMPVERSIONRESPONSE._serialized_end=1516 - _COMPVERSIONRESPONSE_LICENSE._serialized_start=1201 - _COMPVERSIONRESPONSE_LICENSE._serialized_end=1280 - _COMPVERSIONRESPONSE_VERSION._serialized_start=1282 - _COMPVERSIONRESPONSE_VERSION._serialized_end=1382 - _COMPVERSIONRESPONSE_COMPONENT._serialized_start=1385 - _COMPVERSIONRESPONSE_COMPONENT._serialized_end=1516 - _COMPONENTS._serialized_start=1519 - _COMPONENTS._serialized_end=2115 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\337\002\022\235\001\n\032SCANOSS Components Service\022(Provides component intelligence services\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0\032\017api.scanoss.com*\002\002\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007Z8\n6\n\007api_key\022+\010\002\022\032API key for authentication\032\tx-api-key \002b\r\n\013\n\007api_key\022\000' + _globals['_COMPONENTS'].methods_by_name['Echo']._loaded_options = None + _globals['_COMPONENTS'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\022\027/api/v2/components/echo' + _globals['_COMPONENTS'].methods_by_name['SearchComponents']._loaded_options = None + _globals['_COMPONENTS'].methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\033\022\031/api/v2/components/search' + _globals['_COMPONENTS'].methods_by_name['GetComponentVersions']._loaded_options = None + _globals['_COMPONENTS'].methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002\035\022\033/api/v2/components/versions' + _globals['_COMPONENTS'].methods_by_name['GetComponentStatistics']._loaded_options = None + _globals['_COMPONENTS'].methods_by_name['GetComponentStatistics']._serialized_options = b'\202\323\344\223\002\"\"\035/api/v2/components/statistics:\001*' + _globals['_COMPSEARCHREQUEST']._serialized_start=203 + _globals['_COMPSEARCHREQUEST']._serialized_end=321 + _globals['_COMPSTATISTIC']._serialized_start=324 + _globals['_COMPSTATISTIC']._serialized_end=526 + _globals['_COMPSTATISTIC_LANGUAGE']._serialized_start=487 + _globals['_COMPSTATISTIC_LANGUAGE']._serialized_end=526 + _globals['_COMPSTATISTICRESPONSE']._serialized_start=529 + _globals['_COMPSTATISTICRESPONSE']._serialized_end=780 + _globals['_COMPSTATISTICRESPONSE_PURLS']._serialized_start=680 + _globals['_COMPSTATISTICRESPONSE_PURLS']._serialized_end=780 + _globals['_COMPSEARCHRESPONSE']._serialized_start=783 + _globals['_COMPSEARCHRESPONSE']._serialized_end=994 + _globals['_COMPSEARCHRESPONSE_COMPONENT']._serialized_start=937 + _globals['_COMPSEARCHRESPONSE_COMPONENT']._serialized_end=994 + _globals['_COMPVERSIONREQUEST']._serialized_start=996 + _globals['_COMPVERSIONREQUEST']._serialized_end=1045 + _globals['_COMPVERSIONRESPONSE']._serialized_start=1048 + _globals['_COMPVERSIONRESPONSE']._serialized_end=1532 + _globals['_COMPVERSIONRESPONSE_LICENSE']._serialized_start=1203 + _globals['_COMPVERSIONRESPONSE_LICENSE']._serialized_end=1282 + _globals['_COMPVERSIONRESPONSE_VERSION']._serialized_start=1284 + _globals['_COMPVERSIONRESPONSE_VERSION']._serialized_end=1398 + _globals['_COMPVERSIONRESPONSE_COMPONENT']._serialized_start=1401 + _globals['_COMPVERSIONRESPONSE_COMPONENT']._serialized_end=1532 + _globals['_COMPONENTS']._serialized_start=1535 + _globals['_COMPONENTS']._serialized_end=2122 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py index 8ac7b92c..b80082b1 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py @@ -1,10 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.components.v2 import scanoss_components_pb2 as scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2 +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/components/v2/scanoss_components_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class ComponentsStub(object): """ @@ -21,22 +41,22 @@ def __init__(self, channel): '/scanoss.api.components.v2.Components/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.SearchComponents = channel.unary_unary( '/scanoss.api.components.v2.Components/SearchComponents', request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, - ) + _registered_method=True) self.GetComponentVersions = channel.unary_unary( '/scanoss.api.components.v2.Components/GetComponentVersions', request_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, - ) + _registered_method=True) self.GetComponentStatistics = channel.unary_unary( '/scanoss.api.components.v2.Components/GetComponentStatistics', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, - ) + _registered_method=True) class ComponentsServicer(object): @@ -99,6 +119,7 @@ def add_ComponentsServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.components.v2.Components', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.components.v2.Components', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -118,11 +139,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.components.v2.Components/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def SearchComponents(request, @@ -135,11 +166,21 @@ def SearchComponents(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/SearchComponents', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.components.v2.Components/SearchComponents', scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompSearchResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetComponentVersions(request, @@ -152,11 +193,21 @@ def GetComponentVersions(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/GetComponentVersions', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.components.v2.Components/GetComponentVersions', scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompVersionResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetComponentStatistics(request, @@ -169,8 +220,18 @@ def GetComponentStatistics(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.components.v2.Components/GetComponentStatistics', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.components.v2.Components/GetComponentStatistics', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py index f5af2f39..6d488e19 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/cryptography/v2/scanoss-cryptography.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/cryptography/v2/scanoss-cryptography.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,53 +24,53 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 - +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"0\n\tAlgorithm\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\"\xf3\x01\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\x82\x02\n\x19\x41lgorithmsInRangeResponse\x12J\n\x05purls\x18\x01 \x03(\x0b\x32;.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\xe1\x01\n\x17VersionsInRangeResponse\x12H\n\x05purls\x18\x01 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.VersionsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x45\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x15\n\rversions_with\x18\x02 \x03(\t\x12\x18\n\x10versions_without\x18\x03 \x03(\t\"}\n\x04Hint\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08\x63\x61tegory\x18\x04 \x01(\t\x12\x10\n\x03url\x18\x05 \x01(\tH\x00\x88\x01\x01\x12\x11\n\x04purl\x18\x06 \x01(\tH\x01\x88\x01\x01\x42\x06\n\x04_urlB\x07\n\x05_purl\"\xe1\x01\n\rHintsResponse\x12?\n\x05purls\x18\x01 \x03(\x0b\x32\x30.scanoss.api.cryptography.v2.HintsResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\xee\x01\n\x14HintsInRangeResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.cryptography.v2.HintsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint2\x88\x07\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/cryptography/algorithms:\x01*\x12\xa5\x01\n\x14GetAlgorithmsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x36.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse\"1\x82\xd3\xe4\x93\x02+\"&/api/v2/cryptography/algorithmsInRange:\x01*\x12\x9f\x01\n\x12GetVersionsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x34.scanoss.api.cryptography.v2.VersionsInRangeResponse\"/\x82\xd3\xe4\x93\x02)\"$/api/v2/cryptography/versionsInRange:\x01*\x12\x96\x01\n\x0fGetHintsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.cryptography.v2.HintsInRangeResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/cryptography/hintsInRange:\x01*\x12\x8b\x01\n\x12GetEncryptionHints\x12\".scanoss.api.common.v2.PurlRequest\x1a*.scanoss.api.cryptography.v2.HintsResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/cryptography/hints:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"0\n\tAlgorithm\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\"\xf3\x01\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\x82\x02\n\x19\x41lgorithmsInRangeResponse\x12J\n\x05purls\x18\x01 \x03(\x0b\x32;.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\xe1\x01\n\x17VersionsInRangeResponse\x12H\n\x05purls\x18\x01 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.VersionsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x45\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x15\n\rversions_with\x18\x02 \x03(\t\x12\x18\n\x10versions_without\x18\x03 \x03(\t\"b\n\x04Hint\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08\x63\x61tegory\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0c\n\x04purl\x18\x06 \x01(\t\"\xe1\x01\n\rHintsResponse\x12?\n\x05purls\x18\x01 \x03(\x0b\x32\x30.scanoss.api.cryptography.v2.HintsResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\xee\x01\n\x14HintsInRangeResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.cryptography.v2.HintsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint2\x88\x07\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/cryptography/algorithms:\x01*\x12\xa5\x01\n\x14GetAlgorithmsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x36.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse\"1\x82\xd3\xe4\x93\x02+\"&/api/v2/cryptography/algorithmsInRange:\x01*\x12\x9f\x01\n\x12GetVersionsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x34.scanoss.api.cryptography.v2.VersionsInRangeResponse\"/\x82\xd3\xe4\x93\x02)\"$/api/v2/cryptography/versionsInRange:\x01*\x12\x96\x01\n\x0fGetHintsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.cryptography.v2.HintsInRangeResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/cryptography/hintsInRange:\x01*\x12\x8b\x01\n\x12GetEncryptionHints\x12\".scanoss.api.common.v2.PurlRequest\x1a*.scanoss.api.cryptography.v2.HintsResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/cryptography/hints:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _CRYPTOGRAPHY.methods_by_name['Echo']._options = None - _CRYPTOGRAPHY.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/cryptography/echo:\001*' - _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._options = None - _CRYPTOGRAPHY.methods_by_name['GetAlgorithms']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/cryptography/algorithms:\001*' - _CRYPTOGRAPHY.methods_by_name['GetAlgorithmsInRange']._options = None - _CRYPTOGRAPHY.methods_by_name['GetAlgorithmsInRange']._serialized_options = b'\202\323\344\223\002+\"&/api/v2/cryptography/algorithmsInRange:\001*' - _CRYPTOGRAPHY.methods_by_name['GetVersionsInRange']._options = None - _CRYPTOGRAPHY.methods_by_name['GetVersionsInRange']._serialized_options = b'\202\323\344\223\002)\"$/api/v2/cryptography/versionsInRange:\001*' - _CRYPTOGRAPHY.methods_by_name['GetHintsInRange']._options = None - _CRYPTOGRAPHY.methods_by_name['GetHintsInRange']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/cryptography/hintsInRange:\001*' - _CRYPTOGRAPHY.methods_by_name['GetEncryptionHints']._options = None - _CRYPTOGRAPHY.methods_by_name['GetEncryptionHints']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/cryptography/hints:\001*' - _ALGORITHM._serialized_start=207 - _ALGORITHM._serialized_end=255 - _ALGORITHMRESPONSE._serialized_start=258 - _ALGORITHMRESPONSE._serialized_end=501 - _ALGORITHMRESPONSE_PURLS._serialized_start=403 - _ALGORITHMRESPONSE_PURLS._serialized_end=501 - _ALGORITHMSINRANGERESPONSE._serialized_start=504 - _ALGORITHMSINRANGERESPONSE._serialized_end=762 - _ALGORITHMSINRANGERESPONSE_PURL._serialized_start=664 - _ALGORITHMSINRANGERESPONSE_PURL._serialized_end=762 - _VERSIONSINRANGERESPONSE._serialized_start=765 - _VERSIONSINRANGERESPONSE._serialized_end=990 - _VERSIONSINRANGERESPONSE_PURL._serialized_start=921 - _VERSIONSINRANGERESPONSE_PURL._serialized_end=990 - _HINT._serialized_start=992 - _HINT._serialized_end=1117 - _HINTSRESPONSE._serialized_start=1120 - _HINTSRESPONSE._serialized_end=1345 - _HINTSRESPONSE_PURLS._serialized_start=1257 - _HINTSRESPONSE_PURLS._serialized_end=1345 - _HINTSINRANGERESPONSE._serialized_start=1348 - _HINTSINRANGERESPONSE._serialized_end=1586 - _HINTSINRANGERESPONSE_PURL._serialized_start=1498 - _HINTSINRANGERESPONSE_PURL._serialized_end=1586 - _CRYPTOGRAPHY._serialized_start=1589 - _CRYPTOGRAPHY._serialized_end=2493 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_CRYPTOGRAPHY'].methods_by_name['Echo']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/cryptography/echo:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithms']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithms']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/cryptography/algorithms:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithmsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithmsInRange']._serialized_options = b'\202\323\344\223\002+\"&/api/v2/cryptography/algorithmsInRange:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetVersionsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetVersionsInRange']._serialized_options = b'\202\323\344\223\002)\"$/api/v2/cryptography/versionsInRange:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetHintsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetHintsInRange']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/cryptography/hintsInRange:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetEncryptionHints']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetEncryptionHints']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/cryptography/hints:\001*' + _globals['_ALGORITHM']._serialized_start=209 + _globals['_ALGORITHM']._serialized_end=257 + _globals['_ALGORITHMRESPONSE']._serialized_start=260 + _globals['_ALGORITHMRESPONSE']._serialized_end=503 + _globals['_ALGORITHMRESPONSE_PURLS']._serialized_start=405 + _globals['_ALGORITHMRESPONSE_PURLS']._serialized_end=503 + _globals['_ALGORITHMSINRANGERESPONSE']._serialized_start=506 + _globals['_ALGORITHMSINRANGERESPONSE']._serialized_end=764 + _globals['_ALGORITHMSINRANGERESPONSE_PURL']._serialized_start=666 + _globals['_ALGORITHMSINRANGERESPONSE_PURL']._serialized_end=764 + _globals['_VERSIONSINRANGERESPONSE']._serialized_start=767 + _globals['_VERSIONSINRANGERESPONSE']._serialized_end=992 + _globals['_VERSIONSINRANGERESPONSE_PURL']._serialized_start=923 + _globals['_VERSIONSINRANGERESPONSE_PURL']._serialized_end=992 + _globals['_HINT']._serialized_start=994 + _globals['_HINT']._serialized_end=1092 + _globals['_HINTSRESPONSE']._serialized_start=1095 + _globals['_HINTSRESPONSE']._serialized_end=1320 + _globals['_HINTSRESPONSE_PURLS']._serialized_start=1232 + _globals['_HINTSRESPONSE_PURLS']._serialized_end=1320 + _globals['_HINTSINRANGERESPONSE']._serialized_start=1323 + _globals['_HINTSINRANGERESPONSE']._serialized_end=1561 + _globals['_HINTSINRANGERESPONSE_PURL']._serialized_start=1473 + _globals['_HINTSINRANGERESPONSE_PURL']._serialized_end=1561 + _globals['_CRYPTOGRAPHY']._serialized_start=1564 + _globals['_CRYPTOGRAPHY']._serialized_end=2468 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py index 25b77e2c..e015e875 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py @@ -1,10 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.cryptography.v2 import scanoss_cryptography_pb2 as scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2 +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class CryptographyStub(object): """ @@ -21,32 +41,32 @@ def __init__(self, channel): '/scanoss.api.cryptography.v2.Cryptography/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.GetAlgorithms = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, - ) + _registered_method=True) self.GetAlgorithmsInRange = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithmsInRange', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmsInRangeResponse.FromString, - ) + _registered_method=True) self.GetVersionsInRange = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetVersionsInRange', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.VersionsInRangeResponse.FromString, - ) + _registered_method=True) self.GetHintsInRange = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetHintsInRange', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsInRangeResponse.FromString, - ) + _registered_method=True) self.GetEncryptionHints = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetEncryptionHints', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsResponse.FromString, - ) + _registered_method=True) class CryptographyServicer(object): @@ -133,6 +153,7 @@ def add_CryptographyServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -152,11 +173,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetAlgorithms(request, @@ -169,11 +200,21 @@ def GetAlgorithms(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithms', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetAlgorithmsInRange(request, @@ -186,11 +227,21 @@ def GetAlgorithmsInRange(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithmsInRange', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithmsInRange', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmsInRangeResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetVersionsInRange(request, @@ -203,11 +254,21 @@ def GetVersionsInRange(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetVersionsInRange', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetVersionsInRange', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.VersionsInRangeResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetHintsInRange(request, @@ -220,11 +281,21 @@ def GetHintsInRange(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetHintsInRange', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetHintsInRange', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsInRangeResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetEncryptionHints(request, @@ -237,8 +308,18 @@ def GetEncryptionHints(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.cryptography.v2.Cryptography/GetEncryptionHints', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetEncryptionHints', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index a0779b29..59399d84 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/dependencies/v2/scanoss-dependencies.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/dependencies/v2/scanoss-dependencies.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,43 +24,43 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 - +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies\"z\n\x1bTransitiveDependencyRequest\x12\x11\n\tecosystem\x18\x01 \x01(\t\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x12\r\n\x05limit\x18\x03 \x01(\x05\x12*\n\x05purls\x18\x05 \x03(\x0b\x32\x1b.scanoss.api.common.v2.Purl\"\xe2\x01\n\x1cTransitiveDependencyResponse\x12\\\n\x0c\x64\x65pendencies\x18\x01 \x03(\x0b\x32\x46.scanoss.api.dependencies.v2.TransitiveDependencyResponse.Dependencies\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a-\n\x0c\x44\x65pendencies\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t2\xe7\x03\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*\x12\xbc\x01\n\x19GetTransitiveDependencies\x12\x38.scanoss.api.dependencies.v2.TransitiveDependencyRequest\x1a\x39.scanoss.api.dependencies.v2.TransitiveDependencyResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/dependencies/transitive:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies\"z\n\x1bTransitiveDependencyRequest\x12\x11\n\tecosystem\x18\x01 \x01(\t\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x12\r\n\x05limit\x18\x03 \x01(\x05\x12*\n\x05purls\x18\x05 \x03(\x0b\x32\x1b.scanoss.api.common.v2.Purl\"\xe2\x01\n\x1cTransitiveDependencyResponse\x12\\\n\x0c\x64\x65pendencies\x18\x01 \x03(\x0b\x32\x46.scanoss.api.dependencies.v2.TransitiveDependencyResponse.Dependencies\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a-\n\x0c\x44\x65pendencies\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t2\xe7\x03\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*\x12\xbc\x01\n\x19GetTransitiveDependencies\x12\x38.scanoss.api.dependencies.v2.TransitiveDependencyRequest\x1a\x39.scanoss.api.dependencies.v2.TransitiveDependencyResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/dependencies/transitive:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service\"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _DEPENDENCIES.methods_by_name['Echo']._options = None - _DEPENDENCIES.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' - _DEPENDENCIES.methods_by_name['GetDependencies']._options = None - _DEPENDENCIES.methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' - _DEPENDENCIES.methods_by_name['GetTransitiveDependencies']._options = None - _DEPENDENCIES.methods_by_name['GetTransitiveDependencies']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/dependencies/transitive:\001*' - _DEPENDENCYREQUEST._serialized_start=208 - _DEPENDENCYREQUEST._serialized_end=447 - _DEPENDENCYREQUEST_PURLS._serialized_start=313 - _DEPENDENCYREQUEST_PURLS._serialized_end=355 - _DEPENDENCYREQUEST_FILES._serialized_start=357 - _DEPENDENCYREQUEST_FILES._serialized_end=447 - _DEPENDENCYRESPONSE._serialized_start=450 - _DEPENDENCYRESPONSE._serialized_end=986 - _DEPENDENCYRESPONSE_LICENSES._serialized_start=597 - _DEPENDENCYRESPONSE_LICENSES._serialized_end=677 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=680 - _DEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=850 - _DEPENDENCYRESPONSE_FILES._serialized_start=853 - _DEPENDENCYRESPONSE_FILES._serialized_end=986 - _TRANSITIVEDEPENDENCYREQUEST._serialized_start=988 - _TRANSITIVEDEPENDENCYREQUEST._serialized_end=1110 - _TRANSITIVEDEPENDENCYRESPONSE._serialized_start=1113 - _TRANSITIVEDEPENDENCYRESPONSE._serialized_end=1339 - _TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES._serialized_start=1294 - _TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES._serialized_end=1339 - _DEPENDENCIES._serialized_start=1342 - _DEPENDENCIES._serialized_end=1829 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.dependencies.v2.scanoss_dependencies_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service\"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_DEPENDENCIES'].methods_by_name['Echo']._loaded_options = None + _globals['_DEPENDENCIES'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' + _globals['_DEPENDENCIES'].methods_by_name['GetDependencies']._loaded_options = None + _globals['_DEPENDENCIES'].methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' + _globals['_DEPENDENCIES'].methods_by_name['GetTransitiveDependencies']._loaded_options = None + _globals['_DEPENDENCIES'].methods_by_name['GetTransitiveDependencies']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/dependencies/transitive:\001*' + _globals['_DEPENDENCYREQUEST']._serialized_start=210 + _globals['_DEPENDENCYREQUEST']._serialized_end=449 + _globals['_DEPENDENCYREQUEST_PURLS']._serialized_start=315 + _globals['_DEPENDENCYREQUEST_PURLS']._serialized_end=357 + _globals['_DEPENDENCYREQUEST_FILES']._serialized_start=359 + _globals['_DEPENDENCYREQUEST_FILES']._serialized_end=449 + _globals['_DEPENDENCYRESPONSE']._serialized_start=452 + _globals['_DEPENDENCYRESPONSE']._serialized_end=988 + _globals['_DEPENDENCYRESPONSE_LICENSES']._serialized_start=599 + _globals['_DEPENDENCYRESPONSE_LICENSES']._serialized_end=679 + _globals['_DEPENDENCYRESPONSE_DEPENDENCIES']._serialized_start=682 + _globals['_DEPENDENCYRESPONSE_DEPENDENCIES']._serialized_end=852 + _globals['_DEPENDENCYRESPONSE_FILES']._serialized_start=855 + _globals['_DEPENDENCYRESPONSE_FILES']._serialized_end=988 + _globals['_TRANSITIVEDEPENDENCYREQUEST']._serialized_start=990 + _globals['_TRANSITIVEDEPENDENCYREQUEST']._serialized_end=1112 + _globals['_TRANSITIVEDEPENDENCYRESPONSE']._serialized_start=1115 + _globals['_TRANSITIVEDEPENDENCYRESPONSE']._serialized_end=1341 + _globals['_TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES']._serialized_start=1296 + _globals['_TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES']._serialized_end=1341 + _globals['_DEPENDENCIES']._serialized_start=1344 + _globals['_DEPENDENCIES']._serialized_end=1831 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py index ae675b67..aa4e5b7a 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py @@ -1,10 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.dependencies.v2 import scanoss_dependencies_pb2 as scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2 +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class DependenciesStub(object): """ @@ -21,17 +41,17 @@ def __init__(self, channel): '/scanoss.api.dependencies.v2.Dependencies/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.GetDependencies = channel.unary_unary( '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, - ) + _registered_method=True) self.GetTransitiveDependencies = channel.unary_unary( '/scanoss.api.dependencies.v2.Dependencies/GetTransitiveDependencies', request_serializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyResponse.FromString, - ) + _registered_method=True) class DependenciesServicer(object): @@ -82,6 +102,7 @@ def add_DependenciesServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.dependencies.v2.Dependencies', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.dependencies.v2.Dependencies', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -101,11 +122,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.dependencies.v2.Dependencies/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetDependencies(request, @@ -118,11 +149,21 @@ def GetDependencies(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.dependencies.v2.Dependencies/GetDependencies', scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyRequest.SerializeToString, scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.DependencyResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetTransitiveDependencies(request, @@ -135,8 +176,18 @@ def GetTransitiveDependencies(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.dependencies.v2.Dependencies/GetTransitiveDependencies', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.dependencies.v2.Dependencies/GetTransitiveDependencies', scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyRequest.SerializeToString, scanoss_dot_api_dot_dependencies_dot_v2_dot_scanoss__dependencies__pb2.TransitiveDependencyResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py index df9e5754..e79f7ec6 100644 --- a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,37 +24,37 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 - +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto\x12\x1cscanoss.api.geoprovenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xd1\x03\n\x13\x43ontributorResponse\x12\x46\n\x05purls\x18\x01 \x03(\x0b\x32\x37.scanoss.api.geoprovenance.v2.ContributorResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xd3\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12^\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32\x42.scanoss.api.geoprovenance.v2.ContributorResponse.DeclaredLocation\x12\\\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32\x41.scanoss.api.geoprovenance.v2.ContributorResponse.CuratedLocation\"\x99\x02\n\x0eOriginResponse\x12\x41\n\x05purls\x18\x01 \x03(\x0b\x32\x32.scanoss.api.geoprovenance.v2.OriginResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a,\n\x08Location\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\npercentage\x18\x02 \x01(\x02\x1a_\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12H\n\tlocations\x18\x02 \x03(\x0b\x32\x35.scanoss.api.geoprovenance.v2.OriginResponse.Location2\xb9\x03\n\rGeoProvenance\x12v\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/geoprovenance/echo:\x01*\x12\x9d\x01\n\x18GetComponentContributors\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.geoprovenance.v2.ContributorResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/geoprovenance/countries:\x01*\x12\x8f\x01\n\x12GetComponentOrigin\x12\".scanoss.api.common.v2.PurlRequest\x1a,.scanoss.api.geoprovenance.v2.OriginResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/geoprovenance/origin:\x01*B\xa4\x02Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\x92\x41\xe3\x01\x12}\n\x1eSCANOSS GEO Provenance Service\"V\n\x15scanoss-geoprovenance\x12(https://github.com/scanoss/geoprovenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.geoprovenance.v2.scanoss_geoprovenance_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto\x12\x1cscanoss.api.geoprovenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xd1\x03\n\x13\x43ontributorResponse\x12\x46\n\x05purls\x18\x01 \x03(\x0b\x32\x37.scanoss.api.geoprovenance.v2.ContributorResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xd3\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12^\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32\x42.scanoss.api.geoprovenance.v2.ContributorResponse.DeclaredLocation\x12\\\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32\x41.scanoss.api.geoprovenance.v2.ContributorResponse.CuratedLocation\"\x99\x02\n\x0eOriginResponse\x12\x41\n\x05purls\x18\x01 \x03(\x0b\x32\x32.scanoss.api.geoprovenance.v2.OriginResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a,\n\x08Location\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\npercentage\x18\x02 \x01(\x02\x1a_\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12H\n\tlocations\x18\x02 \x03(\x0b\x32\x35.scanoss.api.geoprovenance.v2.OriginResponse.Location2\xb9\x03\n\rGeoProvenance\x12v\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/geoprovenance/echo:\x01*\x12\x9d\x01\n\x18GetComponentContributors\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.geoprovenance.v2.ContributorResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/geoprovenance/countries:\x01*\x12\x8f\x01\n\x12GetComponentOrigin\x12\".scanoss.api.common.v2.PurlRequest\x1a,.scanoss.api.geoprovenance.v2.OriginResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/geoprovenance/origin:\x01*B\xa4\x02Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\x92\x41\xe3\x01\x12}\n\x1eSCANOSS GEO Provenance Service\"V\n\x15scanoss-geoprovenance\x12(https://github.com/scanoss/geoprovenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\222A\343\001\022}\n\036SCANOSS GEO Provenance Service\"V\n\025scanoss-geoprovenance\022(https://github.com/scanoss/geoprovenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _GEOPROVENANCE.methods_by_name['Echo']._options = None - _GEOPROVENANCE.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/geoprovenance/echo:\001*' - _GEOPROVENANCE.methods_by_name['GetComponentContributors']._options = None - _GEOPROVENANCE.methods_by_name['GetComponentContributors']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/geoprovenance/countries:\001*' - _GEOPROVENANCE.methods_by_name['GetComponentOrigin']._options = None - _GEOPROVENANCE.methods_by_name['GetComponentOrigin']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/geoprovenance/origin:\001*' - _CONTRIBUTORRESPONSE._serialized_start=211 - _CONTRIBUTORRESPONSE._serialized_end=676 - _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_start=361 - _CONTRIBUTORRESPONSE_DECLAREDLOCATION._serialized_end=411 - _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_start=413 - _CONTRIBUTORRESPONSE_CURATEDLOCATION._serialized_end=462 - _CONTRIBUTORRESPONSE_PURLS._serialized_start=465 - _CONTRIBUTORRESPONSE_PURLS._serialized_end=676 - _ORIGINRESPONSE._serialized_start=679 - _ORIGINRESPONSE._serialized_end=960 - _ORIGINRESPONSE_LOCATION._serialized_start=819 - _ORIGINRESPONSE_LOCATION._serialized_end=863 - _ORIGINRESPONSE_PURLS._serialized_start=865 - _ORIGINRESPONSE_PURLS._serialized_end=960 - _GEOPROVENANCE._serialized_start=963 - _GEOPROVENANCE._serialized_end=1404 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.geoprovenance.v2.scanoss_geoprovenance_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\222A\343\001\022}\n\036SCANOSS GEO Provenance Service\"V\n\025scanoss-geoprovenance\022(https://github.com/scanoss/geoprovenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_GEOPROVENANCE'].methods_by_name['Echo']._loaded_options = None + _globals['_GEOPROVENANCE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/geoprovenance/echo:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['GetComponentContributors']._loaded_options = None + _globals['_GEOPROVENANCE'].methods_by_name['GetComponentContributors']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/geoprovenance/countries:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['GetComponentOrigin']._loaded_options = None + _globals['_GEOPROVENANCE'].methods_by_name['GetComponentOrigin']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/geoprovenance/origin:\001*' + _globals['_CONTRIBUTORRESPONSE']._serialized_start=213 + _globals['_CONTRIBUTORRESPONSE']._serialized_end=678 + _globals['_CONTRIBUTORRESPONSE_DECLAREDLOCATION']._serialized_start=363 + _globals['_CONTRIBUTORRESPONSE_DECLAREDLOCATION']._serialized_end=413 + _globals['_CONTRIBUTORRESPONSE_CURATEDLOCATION']._serialized_start=415 + _globals['_CONTRIBUTORRESPONSE_CURATEDLOCATION']._serialized_end=464 + _globals['_CONTRIBUTORRESPONSE_PURLS']._serialized_start=467 + _globals['_CONTRIBUTORRESPONSE_PURLS']._serialized_end=678 + _globals['_ORIGINRESPONSE']._serialized_start=681 + _globals['_ORIGINRESPONSE']._serialized_end=962 + _globals['_ORIGINRESPONSE_LOCATION']._serialized_start=821 + _globals['_ORIGINRESPONSE_LOCATION']._serialized_end=865 + _globals['_ORIGINRESPONSE_PURLS']._serialized_start=867 + _globals['_ORIGINRESPONSE_PURLS']._serialized_end=962 + _globals['_GEOPROVENANCE']._serialized_start=965 + _globals['_GEOPROVENANCE']._serialized_end=1406 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py index ff63832a..d669a5e4 100644 --- a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py @@ -1,10 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.geoprovenance.v2 import scanoss_geoprovenance_pb2 as scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2 +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class GeoProvenanceStub(object): """* @@ -21,17 +41,17 @@ def __init__(self, channel): '/scanoss.api.geoprovenance.v2.GeoProvenance/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.GetComponentContributors = channel.unary_unary( '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentContributors', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.FromString, - ) + _registered_method=True) self.GetComponentOrigin = channel.unary_unary( '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentOrigin', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.FromString, - ) + _registered_method=True) class GeoProvenanceServicer(object): @@ -82,6 +102,7 @@ def add_GeoProvenanceServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.geoprovenance.v2.GeoProvenance', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.geoprovenance.v2.GeoProvenance', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -101,11 +122,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.geoprovenance.v2.GeoProvenance/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.geoprovenance.v2.GeoProvenance/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetComponentContributors(request, @@ -118,11 +149,21 @@ def GetComponentContributors(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentContributors', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentContributors', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetComponentOrigin(request, @@ -135,8 +176,18 @@ def GetComponentOrigin(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentOrigin', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentOrigin', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/licenses/__init__.py b/src/scanoss/api/licenses/__init__.py new file mode 100644 index 00000000..1e95c46d --- /dev/null +++ b/src/scanoss/api/licenses/__init__.py @@ -0,0 +1,23 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/licenses/v2/__init__.py b/src/scanoss/api/licenses/v2/__init__.py new file mode 100644 index 00000000..1e95c46d --- /dev/null +++ b/src/scanoss/api/licenses/v2/__init__.py @@ -0,0 +1,23 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" diff --git a/src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py b/src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py new file mode 100644 index 00000000..144f8d7d --- /dev/null +++ b/src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: scanoss/api/licenses/v2/scanoss-licenses.proto +# Protobuf Python Version: 6.31.0 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/licenses/v2/scanoss-licenses.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/licenses/v2/scanoss-licenses.proto\x12\x17scanoss.api.licenses.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xbd\x03\n\x18\x43omponentLicenseResponse\x12@\n\tcomponent\x18\x01 \x01(\x0b\x32-.scanoss.api.licenses.v2.ComponentLicenseInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xa7\x02\x92\x41\xa3\x02\n\xa0\x02J\x9d\x02{\"component\":{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}\"\xe9\x04\n\x19\x43omponentsLicenseResponse\x12\x41\n\ncomponents\x18\x01 \x03(\x0b\x32-.scanoss.api.licenses.v2.ComponentLicenseInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xd1\x03\x92\x41\xcd\x03\n\xca\x03J\xc7\x03{\"components\":[{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py@v1.30.0\",\"requirement\": \"\",\"version\": \"v1.30.0\",\"statement\": \"MIT\", \"licenses\": [{\"id\": \"MIT\",\"full_name\": \"MIT License\"}]} ], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}\"\x89\x01\n\x16LicenseDetailsResponse\x12\x38\n\x07license\x18\x01 \x01(\x0b\x32\'.scanoss.api.licenses.v2.LicenseDetails\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\"\x81\x01\n\x13ObligationsResponse\x12\x33\n\x0bobligations\x18\x01 \x01(\x0b\x32\x1e.scanoss.api.licenses.v2.OSADL\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\"\xa3\x04\n\x04SPDX\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tfull_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65tails_url\x18\x04 \x01(\t\x12\x15\n\rreference_url\x18\x05 \x01(\t\x12\x15\n\ris_deprecated\x18\x06 \x01(\x08\x12\x14\n\x0cis_fsf_libre\x18\x07 \x01(\x08\x12\x17\n\x0fis_osi_approved\x18\x08 \x01(\x08\x12\x10\n\x08see_also\x18\t \x03(\t\x12>\n\ncross_refs\x18\n \x03(\x0b\x32*.scanoss.api.licenses.v2.SPDX.SPDXCrossRef\x12?\n\nexceptions\x18\x0b \x03(\x0b\x32+.scanoss.api.licenses.v2.SPDX.SPDXException\x1a\x88\x01\n\x0cSPDXCrossRef\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x10\n\x08is_valid\x18\x02 \x01(\x08\x12\x0f\n\x07is_live\x18\x03 \x01(\x08\x12\x11\n\ttimestamp\x18\x04 \x01(\t\x12\x17\n\x0fis_wayback_link\x18\x05 \x01(\x08\x12\r\n\x05order\x18\x06 \x01(\x05\x12\r\n\x05match\x18\x07 \x01(\t\x1al\n\rSPDXException\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tfull_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65tails_url\x18\x03 \x01(\t\x12\x10\n\x08see_also\x18\x05 \x03(\t\x12\x15\n\ris_deprecated\x18\x06 \x01(\x08\"\x97\x02\n\x05OSADL\x12\x17\n\x0f\x63opyleft_clause\x18\x01 \x01(\x08\x12\x14\n\x0cpatent_hints\x18\x02 \x01(\x08\x12\x15\n\rcompatibility\x18\x03 \x03(\t\x12\x1f\n\x17\x64\x65pending_compatibility\x18\x04 \x03(\t\x12\x17\n\x0fincompatibility\x18\x05 \x03(\t\x12>\n\tuse_cases\x18\x06 \x03(\x0b\x32+.scanoss.api.licenses.v2.OSADL.OSADLUseCase\x1aN\n\x0cOSADLUseCase\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0fobligation_text\x18\x02 \x01(\t\x12\x17\n\x0fobligation_json\x18\x03 \x01(\t\"{\n\x0bLicenseInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tfull_name\x18\x02 \x01(\t:M\x92\x41J\nHJF{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}\"\xb3\x01\n\x0eLicenseDetails\x12\x11\n\tfull_name\x18\x01 \x01(\t\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32$.scanoss.api.licenses.v2.LicenseType\x12+\n\x04spdx\x18\x03 \x01(\x0b\x32\x1d.scanoss.api.licenses.v2.SPDX\x12-\n\x05osadl\x18\x04 \x01(\x0b\x32\x1e.scanoss.api.licenses.v2.OSADL\"\x1c\n\x0eLicenseRequest\x12\n\n\x02id\x18\x01 \x01(\t\"\x95\x01\n\x14\x43omponentLicenseInfo\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x11\n\tstatement\x18\x04 \x01(\t\x12\x36\n\x08licenses\x18\x05 \x03(\x0b\x32$.scanoss.api.licenses.v2.LicenseInfo*l\n\x0bLicenseType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0e\n\nPERMISSIVE\x10\x01\x12\x0c\n\x08\x43OPYLEFT\x10\x02\x12\x0e\n\nCOMMERCIAL\x10\x03\x12\x0f\n\x0bPROPRIETARY\x10\x04\x12\x11\n\rPUBLIC_DOMAIN\x10\x05\x32\xd0\x05\n\x07License\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/licenses/echo:\x01*\x12\x96\x01\n\x14GetComponentLicenses\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x31.scanoss.api.licenses.v2.ComponentLicenseResponse\"\"\x82\xd3\xe4\x93\x02\x1c\x12\x1a/api/v2/licenses/component\x12\x9d\x01\n\x15GetComponentsLicenses\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x32.scanoss.api.licenses.v2.ComponentsLicenseResponse\"&\x82\xd3\xe4\x93\x02 \"\x1b/api/v2/licenses/components:\x01*\x12\x88\x01\n\nGetDetails\x12\'.scanoss.api.licenses.v2.LicenseRequest\x1a/.scanoss.api.licenses.v2.LicenseDetailsResponse\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/api/v2/licenses/details\x12\x8d\x01\n\x0eGetObligations\x12\'.scanoss.api.licenses.v2.LicenseRequest\x1a,.scanoss.api.licenses.v2.ObligationsResponse\"$\x82\xd3\xe4\x93\x02\x1e\x12\x1c/api/v2/licenses/obligationsB\xa7\x02Z1github.amrom.workers.dev/scanoss/papi/api/licensesv2;licensesv2\x92\x41\xf0\x01\x12\xb4\x01\n\x17SCANOSS License Service\x12\x46License service provides license intelligence for software components.\"L\n\x10scanoss-licenses\x12#https://github.com/scanoss/licenses\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x01\x02\x32\x10\x61pplication/json:\x10\x61pplication/jsonb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.licenses.v2.scanoss_licenses_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z1github.amrom.workers.dev/scanoss/papi/api/licensesv2;licensesv2\222A\360\001\022\264\001\n\027SCANOSS License Service\022FLicense service provides license intelligence for software components.\"L\n\020scanoss-licenses\022#https://github.com/scanoss/licenses\032\023support@scanoss.com2\0032.0\032\017api.scanoss.com*\002\001\0022\020application/json:\020application/json' + _globals['_COMPONENTLICENSERESPONSE']._loaded_options = None + _globals['_COMPONENTLICENSERESPONSE']._serialized_options = b'\222A\243\002\n\240\002J\235\002{\"component\":{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}' + _globals['_COMPONENTSLICENSERESPONSE']._loaded_options = None + _globals['_COMPONENTSLICENSERESPONSE']._serialized_options = b'\222A\315\003\n\312\003J\307\003{\"components\":[{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py@v1.30.0\",\"requirement\": \"\",\"version\": \"v1.30.0\",\"statement\": \"MIT\", \"licenses\": [{\"id\": \"MIT\",\"full_name\": \"MIT License\"}]} ], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}' + _globals['_LICENSEINFO']._loaded_options = None + _globals['_LICENSEINFO']._serialized_options = b'\222AJ\nHJF{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}' + _globals['_LICENSE'].methods_by_name['Echo']._loaded_options = None + _globals['_LICENSE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/licenses/echo:\001*' + _globals['_LICENSE'].methods_by_name['GetComponentLicenses']._loaded_options = None + _globals['_LICENSE'].methods_by_name['GetComponentLicenses']._serialized_options = b'\202\323\344\223\002\034\022\032/api/v2/licenses/component' + _globals['_LICENSE'].methods_by_name['GetComponentsLicenses']._loaded_options = None + _globals['_LICENSE'].methods_by_name['GetComponentsLicenses']._serialized_options = b'\202\323\344\223\002 \"\033/api/v2/licenses/components:\001*' + _globals['_LICENSE'].methods_by_name['GetDetails']._loaded_options = None + _globals['_LICENSE'].methods_by_name['GetDetails']._serialized_options = b'\202\323\344\223\002\032\022\030/api/v2/licenses/details' + _globals['_LICENSE'].methods_by_name['GetObligations']._loaded_options = None + _globals['_LICENSE'].methods_by_name['GetObligations']._serialized_options = b'\202\323\344\223\002\036\022\034/api/v2/licenses/obligations' + _globals['_LICENSETYPE']._serialized_start=2858 + _globals['_LICENSETYPE']._serialized_end=2966 + _globals['_COMPONENTLICENSERESPONSE']._serialized_start=198 + _globals['_COMPONENTLICENSERESPONSE']._serialized_end=643 + _globals['_COMPONENTSLICENSERESPONSE']._serialized_start=646 + _globals['_COMPONENTSLICENSERESPONSE']._serialized_end=1263 + _globals['_LICENSEDETAILSRESPONSE']._serialized_start=1266 + _globals['_LICENSEDETAILSRESPONSE']._serialized_end=1403 + _globals['_OBLIGATIONSRESPONSE']._serialized_start=1406 + _globals['_OBLIGATIONSRESPONSE']._serialized_end=1535 + _globals['_SPDX']._serialized_start=1538 + _globals['_SPDX']._serialized_end=2085 + _globals['_SPDX_SPDXCROSSREF']._serialized_start=1839 + _globals['_SPDX_SPDXCROSSREF']._serialized_end=1975 + _globals['_SPDX_SPDXEXCEPTION']._serialized_start=1977 + _globals['_SPDX_SPDXEXCEPTION']._serialized_end=2085 + _globals['_OSADL']._serialized_start=2088 + _globals['_OSADL']._serialized_end=2367 + _globals['_OSADL_OSADLUSECASE']._serialized_start=2289 + _globals['_OSADL_OSADLUSECASE']._serialized_end=2367 + _globals['_LICENSEINFO']._serialized_start=2369 + _globals['_LICENSEINFO']._serialized_end=2492 + _globals['_LICENSEDETAILS']._serialized_start=2495 + _globals['_LICENSEDETAILS']._serialized_end=2674 + _globals['_LICENSEREQUEST']._serialized_start=2676 + _globals['_LICENSEREQUEST']._serialized_end=2704 + _globals['_COMPONENTLICENSEINFO']._serialized_start=2707 + _globals['_COMPONENTLICENSEINFO']._serialized_end=2856 + _globals['_LICENSE']._serialized_start=2969 + _globals['_LICENSE']._serialized_end=3689 +# @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/licenses/v2/scanoss_licenses_pb2_grpc.py b/src/scanoss/api/licenses/v2/scanoss_licenses_pb2_grpc.py new file mode 100644 index 00000000..ac0df474 --- /dev/null +++ b/src/scanoss/api/licenses/v2/scanoss_licenses_pb2_grpc.py @@ -0,0 +1,302 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 +from scanoss.api.licenses.v2 import scanoss_licenses_pb2 as scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2 + +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/licenses/v2/scanoss_licenses_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class LicenseStub(object): + """ + License Service Definition + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Echo = channel.unary_unary( + '/scanoss.api.licenses.v2.License/Echo', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + _registered_method=True) + self.GetComponentLicenses = channel.unary_unary( + '/scanoss.api.licenses.v2.License/GetComponentLicenses', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ComponentLicenseResponse.FromString, + _registered_method=True) + self.GetComponentsLicenses = channel.unary_unary( + '/scanoss.api.licenses.v2.License/GetComponentsLicenses', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ComponentsLicenseResponse.FromString, + _registered_method=True) + self.GetDetails = channel.unary_unary( + '/scanoss.api.licenses.v2.License/GetDetails', + request_serializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseDetailsResponse.FromString, + _registered_method=True) + self.GetObligations = channel.unary_unary( + '/scanoss.api.licenses.v2.License/GetObligations', + request_serializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ObligationsResponse.FromString, + _registered_method=True) + + +class LicenseServicer(object): + """ + License Service Definition + """ + + def Echo(self, request, context): + """ + Returns the same message that was sent, used for health checks and connectivity testing + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentLicenses(self, request, context): + """ + Get license information for a single software component. + + Examines source code, license files, and package metadata to determine which licenses apply to the component. + Returns license data in both individual SPDX license and SPDX expressions when determinable. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/licenses/v2/README.md?tab=readme-ov-file#getcomponentlicenses + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsLicenses(self, request, context): + """ + Get license information for multiple software components in a single request. + + Examines source code, license files, and package metadata to determine which licenses apply to each component. + Returns license data in both individual SPDX license and SPDX expressions when determinable. + + See https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/licenses/v2/README.md?tab=readme-ov-file#getcomponentslicenses + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetDetails(self, request, context): + """ + Get detailed metadata for a specific license by SPDX identifier. + + Provides comprehensive license information including SPDX registry data, + OSADL compliance metadata, license type classification, and official references. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetObligations(self, request, context): + """ + Get compliance obligations and usage requirements for a specific license. + + Returns structured OSADL compliance data including use cases, obligations, + compatibility information, and patent hints for the specified license. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_LicenseServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.FromString, + response_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.SerializeToString, + ), + 'GetComponentLicenses': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentLicenses, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ComponentLicenseResponse.SerializeToString, + ), + 'GetComponentsLicenses': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsLicenses, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ComponentsLicenseResponse.SerializeToString, + ), + 'GetDetails': grpc.unary_unary_rpc_method_handler( + servicer.GetDetails, + request_deserializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseRequest.FromString, + response_serializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseDetailsResponse.SerializeToString, + ), + 'GetObligations': grpc.unary_unary_rpc_method_handler( + servicer.GetObligations, + request_deserializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseRequest.FromString, + response_serializer=scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ObligationsResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'scanoss.api.licenses.v2.License', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.licenses.v2.License', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class License(object): + """ + License Service Definition + """ + + @staticmethod + def Echo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.licenses.v2.License/Echo', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentLicenses(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.licenses.v2.License/GetComponentLicenses', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ComponentLicenseResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsLicenses(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.licenses.v2.License/GetComponentsLicenses', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ComponentsLicenseResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetDetails(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.licenses.v2.License/GetDetails', + scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseRequest.SerializeToString, + scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseDetailsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetObligations(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.licenses.v2.License/GetObligations', + scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.LicenseRequest.SerializeToString, + scanoss_dot_api_dot_licenses_dot_v2_dot_scanoss__licenses__pb2.ObligationsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 114a0bf8..90ae2840 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/scanning/v2/scanoss-scanning.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/scanning/v2/scanoss-scanning.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,10 +24,10 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\xa3\x03\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a)\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x1a\x94\x01\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0e\n\x06vendor\x18\x03 \x01(\t\x12>\n\x08versions\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHResponse.Version\x12\x0c\n\x04rank\x18\x05 \x01(\x05\x12\r\n\x05order\x18\x06 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\xa3\x03\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a)\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x1a\x94\x01\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0e\n\x06vendor\x18\x03 \x01(\t\x12>\n\x08versions\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHResponse.Version\x12\x0c\n\x04rank\x18\x05 \x01(\x05\x12\r\n\x05order\x18\x06 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -30,20 +41,20 @@ _globals['_SCANNING'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' _globals['_SCANNING'].methods_by_name['FolderHashScan']._loaded_options = None _globals['_SCANNING'].methods_by_name['FolderHashScan']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/scanning/hfh/scan:\001*' - _globals['_HFHREQUEST']._serialized_start=196 - _globals['_HFHREQUEST']._serialized_end=649 - _globals['_HFHREQUEST_CHILDREN']._serialized_start=334 - _globals['_HFHREQUEST_CHILDREN']._serialized_end=649 - _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=596 - _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=649 - _globals['_HFHRESPONSE']._serialized_start=652 - _globals['_HFHRESPONSE']._serialized_end=1071 - _globals['_HFHRESPONSE_VERSION']._serialized_start=784 - _globals['_HFHRESPONSE_VERSION']._serialized_end=825 - _globals['_HFHRESPONSE_COMPONENT']._serialized_start=828 - _globals['_HFHRESPONSE_COMPONENT']._serialized_end=976 - _globals['_HFHRESPONSE_RESULT']._serialized_start=978 - _globals['_HFHRESPONSE_RESULT']._serialized_end=1071 - _globals['_SCANNING']._serialized_start=1074 - _globals['_SCANNING']._serialized_end=1331 + _globals['_HFHREQUEST']._serialized_start=198 + _globals['_HFHREQUEST']._serialized_end=651 + _globals['_HFHREQUEST_CHILDREN']._serialized_start=336 + _globals['_HFHREQUEST_CHILDREN']._serialized_end=651 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=598 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=651 + _globals['_HFHRESPONSE']._serialized_start=654 + _globals['_HFHRESPONSE']._serialized_end=1073 + _globals['_HFHRESPONSE_VERSION']._serialized_start=786 + _globals['_HFHRESPONSE_VERSION']._serialized_end=827 + _globals['_HFHRESPONSE_COMPONENT']._serialized_start=830 + _globals['_HFHRESPONSE_COMPONENT']._serialized_end=978 + _globals['_HFHRESPONSE_RESULT']._serialized_start=980 + _globals['_HFHRESPONSE_RESULT']._serialized_end=1073 + _globals['_SCANNING']._serialized_start=1076 + _globals['_SCANNING']._serialized_end=1333 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py index d00b7e38..6928eed7 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py @@ -1,10 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.scanning.v2 import scanoss_scanning_pb2 as scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2 +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/scanning/v2/scanoss_scanning_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class ScanningStub(object): """* @@ -21,12 +41,12 @@ def __init__(self, channel): '/scanoss.api.scanning.v2.Scanning/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.FolderHashScan = channel.unary_unary( '/scanoss.api.scanning.v2.Scanning/FolderHashScan', request_serializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, - ) + _registered_method=True) class ScanningServicer(object): @@ -65,6 +85,7 @@ def add_ScanningServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.scanning.v2.Scanning', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.scanning.v2.Scanning', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -84,11 +105,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.scanning.v2.Scanning/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def FolderHashScan(request, @@ -101,8 +132,18 @@ def FolderHashScan(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.scanning.v2.Scanning/FolderHashScan', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.scanning.v2.Scanning/FolderHashScan', scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHRequest.SerializeToString, scanoss_dot_api_dot_scanning_dot_v2_dot_scanoss__scanning__pb2.HFHResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py index 1b8b0461..195d4386 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/semgrep/v2/scanoss-semgrep.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/semgrep/v2/scanoss-semgrep.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,29 +24,29 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 - +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a,protoc-gen-swagger/options/annotations.proto\"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12\".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse\"!\x82\xd3\xe4\x93\x02\x1b\"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service\"J\n\x0fscanoss-semgrep\x12\"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.semgrep.v2.scanoss_semgrep_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12\".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse\"!\x82\xd3\xe4\x93\x02\x1b\"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service\"J\n\x0fscanoss-semgrep\x12\"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service\"J\n\017scanoss-semgrep\022\"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _SEMGREP.methods_by_name['Echo']._options = None - _SEMGREP.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\"\024/api/v2/semgrep/echo:\001*' - _SEMGREP.methods_by_name['GetIssues']._options = None - _SEMGREP.methods_by_name['GetIssues']._serialized_options = b'\202\323\344\223\002\033\"\026/api/v2/semgrep/issues:\001*' - _SEMGREPRESPONSE._serialized_start=193 - _SEMGREPRESPONSE._serialized_end=599 - _SEMGREPRESPONSE_ISSUE._serialized_start=329 - _SEMGREPRESPONSE_ISSUE._serialized_end=396 - _SEMGREPRESPONSE_FILE._serialized_start=398 - _SEMGREPRESPONSE_FILE._serialized_end=498 - _SEMGREPRESPONSE_PURLS._serialized_start=500 - _SEMGREPRESPONSE_PURLS._serialized_end=599 - _SEMGREP._serialized_start=602 - _SEMGREP._serialized_end=850 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.semgrep.v2.scanoss_semgrep_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service\"J\n\017scanoss-semgrep\022\"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_SEMGREP'].methods_by_name['Echo']._loaded_options = None + _globals['_SEMGREP'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\"\024/api/v2/semgrep/echo:\001*' + _globals['_SEMGREP'].methods_by_name['GetIssues']._loaded_options = None + _globals['_SEMGREP'].methods_by_name['GetIssues']._serialized_options = b'\202\323\344\223\002\033\"\026/api/v2/semgrep/issues:\001*' + _globals['_SEMGREPRESPONSE']._serialized_start=195 + _globals['_SEMGREPRESPONSE']._serialized_end=601 + _globals['_SEMGREPRESPONSE_ISSUE']._serialized_start=331 + _globals['_SEMGREPRESPONSE_ISSUE']._serialized_end=398 + _globals['_SEMGREPRESPONSE_FILE']._serialized_start=400 + _globals['_SEMGREPRESPONSE_FILE']._serialized_end=500 + _globals['_SEMGREPRESPONSE_PURLS']._serialized_start=502 + _globals['_SEMGREPRESPONSE_PURLS']._serialized_end=601 + _globals['_SEMGREP']._serialized_start=604 + _globals['_SEMGREP']._serialized_end=852 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py index 4748a3ee..fdda3109 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py @@ -1,10 +1,30 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.semgrep.v2 import scanoss_semgrep_pb2 as scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2 +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class SemgrepStub(object): """ @@ -21,12 +41,12 @@ def __init__(self, channel): '/scanoss.api.semgrep.v2.Semgrep/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.GetIssues = channel.unary_unary( '/scanoss.api.semgrep.v2.Semgrep/GetIssues', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, - ) + _registered_method=True) class SemgrepServicer(object): @@ -65,6 +85,7 @@ def add_SemgrepServicer_to_server(servicer, server): generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. @@ -84,11 +105,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.semgrep.v2.Semgrep/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetIssues(request, @@ -101,8 +132,18 @@ def GetIssues(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.semgrep.v2.Semgrep/GetIssues', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.semgrep.v2.Semgrep/GetIssues', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index 9fc87ed3..7608af1d 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'scanoss/api/vulnerabilities/v2/scanoss-vulnerabilities.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,37 +24,73 @@ from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from protoc_gen_swagger.options import annotations_pb2 as protoc__gen__swagger_dot_options_dot_annotations__pb2 - +from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}\"\xbd\t\n\x1f\x43omponentsVulnerabilityResponse\x12N\n\ncomponents\x18\x01 \x03(\x0b\x32:.scanoss.api.vulnerabilities.v2.ComponentVulnerabilityInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x92\x08\x92\x41\x8e\x08\n\x8b\x08J\x88\x08{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \"1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\",\"requirement\": \"v1.30.0\",\"version\": \"v1.30.0\", \"vulnerabilities\": [{\"id\": \"CVE-2024-54321\", \"cve\": \"CVE-2024-54321\", \"url\": \"https://nvd.nist.gov/vuln/detail/CVE-2024-54321\", \"summary\": \"Denial of service vulnerability\", \"severity\": \"Medium\", \"published\": \"2024-01-15\", \"modified\": \"2024-02-01\", \"source\": \"NDV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L\", \"cvss_score\": 4.3, \"cvss_severity\": \"Medium\"}]}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}2\xc7\x08\n\x0fVulnerabilities\x12x\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/vulnerabilities/echo:\x01*\x12q\n\x07GetCpes\x12\x34.scanoss.api.vulnerabilities.v2.VulnerabilityRequest\x1a+.scanoss.api.vulnerabilities.v2.CpeResponse\"\x03\x88\x02\x01\x12\xa2\x01\n\x10GetComponentCpes\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x35.scanoss.api.vulnerabilities.v2.ComponentCpesResponse\".\x82\xd3\xe4\x93\x02(\x12&/api/v2/vulnerabilities/cpes/component\x12\xa9\x01\n\x11GetComponentsCpes\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x36.scanoss.api.vulnerabilities.v2.ComponentsCpesResponse\"2\x82\xd3\xe4\x93\x02,\"\'/api/v2/vulnerabilities/cpes/components:\x01*\x12\x86\x01\n\x12GetVulnerabilities\x12\x34.scanoss.api.vulnerabilities.v2.VulnerabilityRequest\x1a\x35.scanoss.api.vulnerabilities.v2.VulnerabilityResponse\"\x03\x88\x02\x01\x12\xb1\x01\n\x1bGetComponentVulnerabilities\x12\'.scanoss.api.common.v2.ComponentRequest\x1a>.scanoss.api.vulnerabilities.v2.ComponentVulnerabilityResponse\")\x82\xd3\xe4\x93\x02#\x12!/api/v2/vulnerabilities/component\x12\xb8\x01\n\x1cGetComponentsVulnerabilities\x12(.scanoss.api.common.v2.ComponentsRequest\x1a?.scanoss.api.vulnerabilities.v2.ComponentsVulnerabilityResponse\"-\x82\xd3\xe4\x93\x02\'\"\"/api/v2/vulnerabilities/components:\x01*B\x92\x03Z?github.com/scanoss/papi/api/vulnerabilitiesv2;vulnerabilitiesv2\x92\x41\xcd\x02\x12\xd4\x01\n\x1dSCANOSS Vulnerability Service\x12RVulnerability service provides vulnerability intelligence for software components.\"Z\n\x17scanoss-vulnerabilities\x12*https://github.com/scanoss/vulnerabilities\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x01\x02\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'Z?github.com/scanoss/papi/api/vulnerabilitiesv2;vulnerabilitiesv2\222A\347\001\022\200\001\n\035SCANOSS Vulnerability Service\"Z\n\027scanoss-vulnerabilities\022*https://github.com/scanoss/vulnerabilities\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' - _VULNERABILITIES.methods_by_name['Echo']._options = None - _VULNERABILITIES.methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/vulnerabilities/echo:\001*' - _VULNERABILITIES.methods_by_name['GetCpes']._options = None - _VULNERABILITIES.methods_by_name['GetCpes']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/vulnerabilities/cpes:\001*' - _VULNERABILITIES.methods_by_name['GetVulnerabilities']._options = None - _VULNERABILITIES.methods_by_name['GetVulnerabilities']._serialized_options = b'\202\323\344\223\002,\"\'/api/v2/vulnerabilities/vulnerabilities:\001*' - _VULNERABILITYREQUEST._serialized_start=217 - _VULNERABILITYREQUEST._serialized_end=358 - _VULNERABILITYREQUEST_PURLS._serialized_start=316 - _VULNERABILITYREQUEST_PURLS._serialized_end=358 - _CPERESPONSE._serialized_start=361 - _CPERESPONSE._serialized_end=532 - _CPERESPONSE_PURLS._serialized_start=497 - _CPERESPONSE_PURLS._serialized_end=532 - _VULNERABILITYRESPONSE._serialized_start=535 - _VULNERABILITYRESPONSE._serialized_end=954 - _VULNERABILITYRESPONSE_VULNERABILITIES._serialized_start=692 - _VULNERABILITYRESPONSE_VULNERABILITIES._serialized_end=835 - _VULNERABILITYRESPONSE_PURLS._serialized_start=837 - _VULNERABILITYRESPONSE_PURLS._serialized_end=954 - _VULNERABILITIES._serialized_start=957 - _VULNERABILITIES._serialized_end=1432 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.vulnerabilities.v2.scanoss_vulnerabilities_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'Z?github.com/scanoss/papi/api/vulnerabilitiesv2;vulnerabilitiesv2\222A\315\002\022\324\001\n\035SCANOSS Vulnerability Service\022RVulnerability service provides vulnerability intelligence for software components.\"Z\n\027scanoss-vulnerabilities\022*https://github.com/scanoss/vulnerabilities\032\023support@scanoss.com2\0032.0\032\017api.scanoss.com*\002\001\0022\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_VULNERABILITYREQUEST']._loaded_options = None + _globals['_VULNERABILITYREQUEST']._serialized_options = b'\030\001' + _globals['_CPERESPONSE']._loaded_options = None + _globals['_CPERESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTCPESRESPONSE']._loaded_options = None + _globals['_COMPONENTCPESRESPONSE']._serialized_options = b'\222A\360\001\n\355\001J\352\001{\"component\":{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"1.0.0\", \"version\": \"1.0.0\", \"cpes\": [\"cpe:2.3:a:scanoss:engine:1.0.0:*:*:*:*:*:*:*\"]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"CPEs Successfully retrieved\"}}' + _globals['_COMPONENTSCPESRESPONSE']._loaded_options = None + _globals['_COMPONENTSCPESRESPONSE']._serialized_options = b'\222A\212\003\n\207\003J\204\003{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement=\": \"1.0.0\", \"version=\": \"1.0.0\", \"cpes\": [\"cpe:2.3:a:scanoss:engine:1.0.0:*:*:*:*:*:*:*\"]}, {\"purl\": \"pkg:github/scanoss/scanoss.py@v1.30.0\",\"requirement\": \"\",\"version\": \"v1.30.0\", \"cpes\": [\"cpe:2.3:a:scanoss:scanoss.py:1.30.0:*:*:*:*:*:*:*\"]} ], \"status\": {\"status\": \"SUCCESS\", \"message\": \"CPEs Successfully retrieved\"}}' + _globals['_VULNERABILITYRESPONSE']._loaded_options = None + _globals['_VULNERABILITYRESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTVULNERABILITYRESPONSE']._loaded_options = None + _globals['_COMPONENTVULNERABILITYRESPONSE']._serialized_options = b'\222A\266\004\n\263\004J\260\004{\"component\":{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \"=>1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}' + _globals['_COMPONENTSVULNERABILITYRESPONSE']._loaded_options = None + _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_options = b'\222A\216\010\n\213\010J\210\010{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \"1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\",\"requirement\": \"v1.30.0\",\"version\": \"v1.30.0\", \"vulnerabilities\": [{\"id\": \"CVE-2024-54321\", \"cve\": \"CVE-2024-54321\", \"url\": \"https://nvd.nist.gov/vuln/detail/CVE-2024-54321\", \"summary\": \"Denial of service vulnerability\", \"severity\": \"Medium\", \"published\": \"2024-01-15\", \"modified\": \"2024-02-01\", \"source\": \"NDV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L\", \"cvss_score\": 4.3, \"cvss_severity\": \"Medium\"}]}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}' + _globals['_VULNERABILITIES'].methods_by_name['Echo']._loaded_options = None + _globals['_VULNERABILITIES'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/vulnerabilities/echo:\001*' + _globals['_VULNERABILITIES'].methods_by_name['GetCpes']._loaded_options = None + _globals['_VULNERABILITIES'].methods_by_name['GetCpes']._serialized_options = b'\210\002\001' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentCpes']._loaded_options = None + _globals['_VULNERABILITIES'].methods_by_name['GetComponentCpes']._serialized_options = b'\202\323\344\223\002(\022&/api/v2/vulnerabilities/cpes/component' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentsCpes']._loaded_options = None + _globals['_VULNERABILITIES'].methods_by_name['GetComponentsCpes']._serialized_options = b'\202\323\344\223\002,\"\'/api/v2/vulnerabilities/cpes/components:\001*' + _globals['_VULNERABILITIES'].methods_by_name['GetVulnerabilities']._loaded_options = None + _globals['_VULNERABILITIES'].methods_by_name['GetVulnerabilities']._serialized_options = b'\210\002\001' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentVulnerabilities']._loaded_options = None + _globals['_VULNERABILITIES'].methods_by_name['GetComponentVulnerabilities']._serialized_options = b'\202\323\344\223\002#\022!/api/v2/vulnerabilities/component' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentsVulnerabilities']._loaded_options = None + _globals['_VULNERABILITIES'].methods_by_name['GetComponentsVulnerabilities']._serialized_options = b'\202\323\344\223\002\'\"\"/api/v2/vulnerabilities/components:\001*' + _globals['_VULNERABILITYREQUEST']._serialized_start=219 + _globals['_VULNERABILITYREQUEST']._serialized_end=364 + _globals['_VULNERABILITYREQUEST_PURLS']._serialized_start=318 + _globals['_VULNERABILITYREQUEST_PURLS']._serialized_end=360 + _globals['_CPERESPONSE']._serialized_start=367 + _globals['_CPERESPONSE']._serialized_end=542 + _globals['_CPERESPONSE_PURLS']._serialized_start=503 + _globals['_CPERESPONSE_PURLS']._serialized_end=538 + _globals['_COMPONENTCPESINFO']._serialized_start=544 + _globals['_COMPONENTCPESINFO']._serialized_end=629 + _globals['_COMPONENTCPESRESPONSE']._serialized_start=632 + _globals['_COMPONENTCPESRESPONSE']._serialized_end=1027 + _globals['_COMPONENTSCPESRESPONSE']._serialized_start=1030 + _globals['_COMPONENTSCPESRESPONSE']._serialized_end=1581 + _globals['_CVSS']._serialized_start=1583 + _globals['_CVSS']._serialized_end=1646 + _globals['_VULNERABILITY']._serialized_start=1649 + _globals['_VULNERABILITY']._serialized_end=1842 + _globals['_VULNERABILITYRESPONSE']._serialized_start=1845 + _globals['_VULNERABILITYRESPONSE']._serialized_end=2098 + _globals['_VULNERABILITYRESPONSE_PURLS']._serialized_start=2001 + _globals['_VULNERABILITYRESPONSE_PURLS']._serialized_end=2094 + _globals['_COMPONENTVULNERABILITYINFO']._serialized_start=2101 + _globals['_COMPONENTVULNERABILITYINFO']._serialized_end=2253 + _globals['_COMPONENTVULNERABILITYRESPONSE']._serialized_start=2256 + _globals['_COMPONENTVULNERABILITYRESPONSE']._serialized_end=2995 + _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_start=2998 + _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_end=4211 + _globals['_VULNERABILITIES']._serialized_start=4214 + _globals['_VULNERABILITIES']._serialized_end=5309 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py index 59e7b6db..158aaf6b 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py @@ -1,14 +1,34 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings from scanoss.api.common.v2 import scanoss_common_pb2 as scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2 from scanoss.api.vulnerabilities.v2 import scanoss_vulnerabilities_pb2 as scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2 +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + class VulnerabilitiesStub(object): """ - Expose all of the SCANOSS Vulnerability RPCs here + Vulnerability Service Definition """ def __init__(self, channel): @@ -21,40 +41,125 @@ def __init__(self, channel): '/scanoss.api.vulnerabilities.v2.Vulnerabilities/Echo', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - ) + _registered_method=True) self.GetCpes = channel.unary_unary( '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetCpes', request_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.CpeResponse.FromString, - ) + _registered_method=True) + self.GetComponentCpes = channel.unary_unary( + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentCpes', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentCpesResponse.FromString, + _registered_method=True) + self.GetComponentsCpes = channel.unary_unary( + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentsCpes', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentsCpesResponse.FromString, + _registered_method=True) self.GetVulnerabilities = channel.unary_unary( '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetVulnerabilities', request_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityResponse.FromString, - ) + _registered_method=True) + self.GetComponentVulnerabilities = channel.unary_unary( + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentVulnerabilities', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentVulnerabilityResponse.FromString, + _registered_method=True) + self.GetComponentsVulnerabilities = channel.unary_unary( + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentsVulnerabilities', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentsVulnerabilityResponse.FromString, + _registered_method=True) class VulnerabilitiesServicer(object): """ - Expose all of the SCANOSS Vulnerability RPCs here + Vulnerability Service Definition """ def Echo(self, request, context): - """Standard echo + """ + Returns the same message that was sent, used for health checks and connectivity testing """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetCpes(self, request, context): - """Get CPEs associated with a PURL + """ + Get CPEs (Common Platform Enumeration) associated with a PURL - legacy endpoint. + + Legacy method for retrieving Common Platform Enumeration identifiers + associated with software components. Use GetComponentCpes instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentCpes(self, request, context): + """ + Get CPEs (Common Platform Enumeration) associated with a single software component. + + Returns Common Platform Enumeration identifiers that match the specified component. + CPEs are used to identify IT platforms in vulnerability databases and enable + vulnerability scanning and assessment. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/vulnerabilities/v2/README.md?tab=readme-ov-file#getcomponentcpes + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsCpes(self, request, context): + """ + Get CPEs (Common Platform Enumeration) associated with multiple software components. + + Returns Common Platform Enumeration identifiers for multiple components in a single request. + CPEs are used to identify IT platforms in vulnerability databases and enable + vulnerability scanning and assessment. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/vulnerabilities/v2/README.md?tab=readme-ov-file#getcomponentscpes """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetVulnerabilities(self, request, context): - """Get vulnerability details + """ + Get vulnerability details - legacy endpoint. + + Legacy method for retrieving vulnerability information for software components. + Use GetComponentVulnerabilities or GetComponentsVulnerabilities instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentVulnerabilities(self, request, context): + """ + Get vulnerability information for a single software component. + + Analyzes the component and returns known vulnerabilities including CVE details, + severity scores, publication dates, and other security metadata. + Vulnerability data is sourced from various security databases and feeds. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/vulnerabilities/v2/README.md?tab=readme-ov-file#getcomponentvulnerabilities + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsVulnerabilities(self, request, context): + """ + Get vulnerability information for multiple software components in a single request. + + Analyzes multiple components and returns known vulnerabilities for each including CVE details, + severity scores, publication dates, and other security metadata. + Vulnerability data is sourced from various security databases and feeds. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/vulnerabilities/v2/README.md?tab=readme-ov-file#getcomponentsvulnerabilities """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -73,21 +178,42 @@ def add_VulnerabilitiesServicer_to_server(servicer, server): request_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityRequest.FromString, response_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.CpeResponse.SerializeToString, ), + 'GetComponentCpes': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentCpes, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentCpesResponse.SerializeToString, + ), + 'GetComponentsCpes': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsCpes, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentsCpesResponse.SerializeToString, + ), 'GetVulnerabilities': grpc.unary_unary_rpc_method_handler( servicer.GetVulnerabilities, request_deserializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityRequest.FromString, response_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityResponse.SerializeToString, ), + 'GetComponentVulnerabilities': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentVulnerabilities, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentVulnerabilityResponse.SerializeToString, + ), + 'GetComponentsVulnerabilities': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsVulnerabilities, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentsVulnerabilityResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.vulnerabilities.v2.Vulnerabilities', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('scanoss.api.vulnerabilities.v2.Vulnerabilities', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class Vulnerabilities(object): """ - Expose all of the SCANOSS Vulnerability RPCs here + Vulnerability Service Definition """ @staticmethod @@ -101,11 +227,21 @@ def Echo(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.vulnerabilities.v2.Vulnerabilities/Echo', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/Echo', scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoRequest.SerializeToString, scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.EchoResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetCpes(request, @@ -118,11 +254,75 @@ def GetCpes(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetCpes', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetCpes', scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityRequest.SerializeToString, scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.CpeResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentCpes(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentCpes', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentCpesResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsCpes(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentsCpes', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentsCpesResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) @staticmethod def GetVulnerabilities(request, @@ -135,8 +335,72 @@ def GetVulnerabilities(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetVulnerabilities', + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetVulnerabilities', scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityRequest.SerializeToString, scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.VulnerabilityResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentVulnerabilities(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentVulnerabilities', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentVulnerabilityResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsVulnerabilities(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.vulnerabilities.v2.Vulnerabilities/GetComponentsVulnerabilities', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_vulnerabilities_dot_v2_dot_scanoss__vulnerabilities__pb2.ComponentsVulnerabilityResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 7e447522..c3aee759 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -796,9 +796,9 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_inspect_dt_project_violation.add_argument( '--format', '-f', required=False, - choices=['json', 'md'], + choices=['json', 'md', 'jira_md'], default='json', - help='Output format: json (default) or md (Markdown)' + help='Output format: json (default), md (Markdown) or jira_md (JIRA Markdown)' ) p_inspect_dt_project_violation.add_argument( '--timeout', '-M', diff --git a/src/scanoss/inspection/dependency_track/project_violation.py b/src/scanoss/inspection/dependency_track/project_violation.py index 9e2d5ed2..b1c7597c 100644 --- a/src/scanoss/inspection/dependency_track/project_violation.py +++ b/src/scanoss/inspection/dependency_track/project_violation.py @@ -34,7 +34,6 @@ DEFAULT_TIME_OUT = 300.0 MILLISECONDS_TO_SECONDS = 1000 - """ Dependency Track project violation policy check implementation. @@ -43,6 +42,7 @@ for a specific project. """ + class ResolvedLicenseDict(TypedDict): """TypedDict for resolved license information from Dependency Track.""" uuid: str @@ -125,7 +125,7 @@ class DependencyTrackProjectViolationPolicyCheck(PolicyCheck[PolicyViolationDict This class handles retrieving, processing, and formatting policy violations from a Dependency Track instance for a specific project. """ - + def __init__( # noqa: PLR0913 self, debug: bool = False, @@ -161,13 +161,13 @@ def __init__( # noqa: PLR0913 timeout: Timeout for processing in seconds (default: 300) """ super().__init__(debug, trace, quiet, format_type, status, 'dependency-track', output) - self.url = url self.api_key = api_key self.project_id = project_id self.project_name = project_name self.project_version = project_version self.upload_token = upload_token self.timeout = timeout + self.url = url.strip().rstrip('/') if url else None self.dep_track_service = DependencyTrackService(self.api_key, self.url, debug=debug, trace=trace, quiet=quiet) def _json(self, project_violations: list[PolicyViolationDict]) -> Dict[str, Any]: @@ -230,7 +230,7 @@ def is_project_updated(self, dt_project: Dict[str, Any]) -> bool: if not dt_project: self.print_stderr('Warning: No project details supplied. Returning False.') return False - + # Safely extract and normalise timestamp values to numeric types def _safe_timestamp(field, value=None, default=0) -> float: """Convert timestamp value to float, handling string/numeric types safely.""" @@ -241,7 +241,7 @@ def _safe_timestamp(field, value=None, default=0) -> float: except (ValueError, TypeError): self.print_stderr(f'Warning: Invalid timestamp for {field}, value: {value}, using default: {default}') return float(default) - + last_import = _safe_timestamp('lastBomImport', dt_project.get('lastBomImport'), 0) last_vulnerability_analysis = _safe_timestamp('lastVulnerabilityAnalysis', dt_project.get('lastVulnerabilityAnalysis'), 0 @@ -372,7 +372,7 @@ def _sort_project_violations(violations: List[PolicyViolationDict]) -> List[Poli """ type_priority = {'SECURITY': 3, 'LICENSE': 2, 'OTHER': 1} return sorted( - violations, + violations, key=lambda x: -type_priority.get(x.get('type', 'OTHER'), 1) ) @@ -424,8 +424,9 @@ def _md_summary_generator(self, project_violations: list[PolicyViolationDict], t rows.append(row) # End for loop return { - "details": f'### Dependency Track Project Violations\n{table_generator(headers, rows, c_cols)}\n', - "summary": f'{len(project_violations)} policy violations were found.\n', + "details": f'### Dependency Track Project Violations\n{table_generator(headers, rows, c_cols)}\n\n' + f'View project in Dependency Track [here]({self.url}/projects/{self.project_id}).\n', + "summary": f'{len(project_violations)} policy violations were found.\n' } def run(self) -> int: diff --git a/src/scanoss/services/dependency_track_service.py b/src/scanoss/services/dependency_track_service.py index 7a367a0f..debc287d 100644 --- a/src/scanoss/services/dependency_track_service.py +++ b/src/scanoss/services/dependency_track_service.py @@ -41,7 +41,7 @@ def __init__( super().__init__(debug=debug, trace=trace, quiet=quiet) if not url: raise ValueError("Error: Dependency Track URL is required") - self.url = url.rstrip('/') + self.url = url.strip().rstrip('/') if not api_key: raise ValueError("Error: Dependency Track API key is required") self.api_key = api_key diff --git a/src/scanoss/threadeddependencies.py b/src/scanoss/threadeddependencies.py index b2cff5d1..c083f4f4 100644 --- a/src/scanoss/threadeddependencies.py +++ b/src/scanoss/threadeddependencies.py @@ -22,12 +22,12 @@ THE SOFTWARE. """ -import threading -import queue import json -from enum import Enum -from typing import Dict, Optional, Set +import queue +import threading from dataclasses import dataclass +from enum import Enum +from typing import Dict from .scancodedeps import ScancodeDeps from .scanossbase import ScanossBase @@ -63,7 +63,7 @@ class ThreadedDependencies(ScanossBase): inputs: queue.Queue = queue.Queue() output: queue.Queue = queue.Queue() - def __init__( + def __init__( # noqa: PLR0913 self, sc_deps: ScancodeDeps, grpc_api: ScanossGrpc, @@ -180,13 +180,15 @@ def filter_dependencies_by_scopes( return self.filter_dependencies( deps, lambda purl: (exclude and purl not in exclude) or (not exclude and purl in include) ) + return None - def scan_dependencies( + def scan_dependencies( # noqa: PLR0912 self, dep_scope: SCOPE = None, dep_scope_include: str = None, dep_scope_exclude: str = None ) -> None: """ Scan for dependencies from the given file/dir or from an input file (from the input queue). """ + # TODO refactor to simplify branches based on PLR0912 current_thread = threading.get_ident() self.print_trace(f'Starting dependency worker {current_thread}...') try: @@ -194,18 +196,17 @@ def scan_dependencies( deps = None if what_to_scan.startswith(DEP_FILE_PREFIX): # We have a pre-parsed dependency file, load it deps = self.sc_deps.load_from_file(what_to_scan.strip(DEP_FILE_PREFIX)) - else: # Search the file/folder for dependency files to parse - if not self.sc_deps.run_scan(what_to_scan=what_to_scan): - self._errors = True - else: - deps = self.sc_deps.produce_from_file() - if dep_scope is not None: - self.print_debug(f'Filtering {dep_scope.name} dependencies') - if dep_scope_include is not None: - self.print_debug(f"Including dependencies with '{dep_scope_include.split(',')}' scopes") - if dep_scope_exclude is not None: - self.print_debug(f"Excluding dependencies with '{dep_scope_exclude.split(',')}' scopes") - deps = self.filter_dependencies_by_scopes(deps, dep_scope, dep_scope_include, dep_scope_exclude) + elif not self.sc_deps.run_scan(what_to_scan=what_to_scan): + self._errors = True + else: + deps = self.sc_deps.produce_from_file() + if dep_scope is not None: + self.print_debug(f'Filtering {dep_scope.name} dependencies') + if dep_scope_include is not None: + self.print_debug(f"Including dependencies with '{dep_scope_include.split(',')}' scopes") + if dep_scope_exclude is not None: + self.print_debug(f"Excluding dependencies with '{dep_scope_exclude.split(',')}' scopes") + deps = self.filter_dependencies_by_scopes(deps, dep_scope, dep_scope_include, dep_scope_exclude) if not self._errors: if deps is None: From 84eee424c6d6b6b7cb9106abbc57cdc9250c113a Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Mon, 1 Sep 2025 13:14:33 +0100 Subject: [PATCH 389/489] Add REST support for dependencies and vulnerabilities * add REST support for dependencies and vulnerabilities * updated scanoss-py to v1.32.0 * fix issue with api key as evn --- CHANGELOG.md | 7 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 7 +- src/scanoss/components.py | 35 +++-- src/scanoss/scanner.py | 3 + src/scanoss/scanossapi.py | 46 ++++--- src/scanoss/scanossgrpc.py | 260 ++++++++++++++++++++++++++++--------- 7 files changed, 261 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62180f9a..fb3d1d38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.32.0] - 2025-09-01 +### Added +- Switched vulnerability and dependency APIs to use REST by default + ## [1.31.5] - 2025-08-27 ### Added - Added jira markdown option for DT @@ -655,4 +659,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.31.2]: https://github.com/scanoss/scanoss.py/compare/v1.31.1...v1.31.2 [1.31.3]: https://github.com/scanoss/scanoss.py/compare/v1.31.2...v1.31.3 [1.31.4]: https://github.com/scanoss/scanoss.py/compare/v1.31.3...v1.31.4 -[1.31.5]: https://github.com/scanoss/scanoss.py/compare/v1.31.4...v1.31.5 \ No newline at end of file +[1.31.5]: https://github.com/scanoss/scanoss.py/compare/v1.31.4...v1.31.5 +[1.31.5]: https://github.com/scanoss/scanoss.py/compare/v1.31.5...v1.32.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index d37886ff..77eed9d1 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.31.5' +__version__ = '1.32.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index c3aee759..983d773d 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -308,6 +308,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Retrieve vulnerabilities for the given components', ) c_vulns.set_defaults(func=comp_vulns) + c_vulns.add_argument('--grpc', action='store_true', help='Enable gRPC support') # Component Sub-command: component semgrep c_semgrep = comp_sub.add_parser( @@ -964,7 +965,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p.add_argument( '--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)' ) - p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') + p.add_argument('--grpc', action='store_true', help='Enable gRPC support') # Global Scan/Fingerprint filter options for p in [p_scan, p_wfp]: @@ -1055,6 +1056,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 type=str, help='Headers to be sent on request (e.g., -hdr "Name: Value") - can be used multiple times', ) + p.add_argument('--ignore-cert-errors', action='store_true', help='Ignore certificate errors') # Syft options for p in [p_cs, p_dep]: @@ -1418,6 +1420,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, req_headers=process_req_headers(args.header), + use_grpc=args.grpc ) if args.wfp: if not scanner.is_file_or_snippet_scan(): @@ -2144,6 +2147,8 @@ def comp_vulns(parser, args): pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), + ignore_cert_errors=args.ignore_cert_errors, + use_grpc=args.grpc, ) if not comps.get_vulnerabilities(args.input, args.purl, args.output): sys.exit(1) diff --git a/src/scanoss/components.py b/src/scanoss/components.py index c68a2336..6756fc82 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -52,6 +52,8 @@ def __init__( # noqa: PLR0913, PLR0915 ca_cert: str = None, pac: PACFile = None, req_headers: dict = None, + ignore_cert_errors: bool = False, + use_grpc: bool = False, ): """ Handle all component style requests @@ -66,6 +68,9 @@ def __init__( # noqa: PLR0913, PLR0915 :param grpc_proxy: Specific gRPC proxy (optional) :param ca_cert: TLS client certificate (optional) :param pac: Proxy Auto-Config file (optional) + :param req_headers: Additional headers to send with requests (optional) + :param ignore_cert_errors: Ignore TLS certificate errors (optional) + :param use_grpc: Use gRPC instead of HTTP (optional) """ super().__init__(debug, trace, quiet) ver_details = Scanner.version_details() @@ -82,14 +87,28 @@ def __init__( # noqa: PLR0913, PLR0915 grpc_proxy=grpc_proxy, timeout=timeout, req_headers=req_headers, + ignore_cert_errors=ignore_cert_errors, + use_grpc=use_grpc, ) - def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None) -> Optional[dict]: + def load_comps(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None)-> Optional[dict]: + """ + Load the specified components and return a dictionary + + :param json_file: JSON Components file (optional) + :param purls: list pf PURLs (optional) + :return: Components Request dictionary or None + """ + return self.load_purls(json_file, purls, 'components') + + def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None, field:str = 'purls' + ) -> Optional[dict]: """ Load the specified purls and return a dictionary :param json_file: JSON PURL file (optional) :param purls: list of PURLs (optional) + :param field: Name of the dictionary field to store the purls in (default: 'purls') :return: PURL Request dictionary or None """ if json_file: @@ -109,14 +128,14 @@ def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] parsed_purls = [] for p in purls: parsed_purls.append({'purl': p}) - purl_request = {'purls': parsed_purls} + purl_request = {field: parsed_purls} else: self.print_stderr('ERROR: No purls specified to process.') return None - purl_count = len(purl_request.get('purls', [])) - self.print_debug(f'Parsed Purls ({purl_count}): {purl_request}') + purl_count = len(purl_request.get(field, [])) + self.print_debug(f'Parsed {field} ({purl_count}): {purl_request}') if purl_count == 0: - self.print_stderr('ERROR: No PURLs parsed from request.') + self.print_stderr(f'ERROR: No {field} parsed from request.') return None return purl_request @@ -142,8 +161,8 @@ def _open_file_or_sdtout(self, filename): """ Open the given filename if requested, otherwise return STDOUT - :param filename: - :return: + :param filename: filename to open or None to return STDOUT + :return: file descriptor or None """ file = sys.stdout if filename: @@ -202,7 +221,7 @@ def get_vulnerabilities(self, json_file: str = None, purls: [] = None, output_fi :return: True on success, False otherwise """ success = False - purls_request = self.load_purls(json_file, purls) + purls_request = self.load_comps(json_file, purls) if purls_request is None or len(purls_request) == 0: return False file = self._open_file_or_sdtout(output_file) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 76767ce7..63803e54 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -107,6 +107,7 @@ def __init__( # noqa: PLR0913, PLR0915 skip_md5_ids=None, scan_settings: 'ScanossSettings | None' = None, req_headers: dict = None, + use_grpc: bool = False, ): """ Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning @@ -173,6 +174,8 @@ def __init__( # noqa: PLR0913, PLR0915 pac=pac, grpc_proxy=grpc_proxy, req_headers=self.req_headers, + ignore_cert_errors=ignore_cert_errors, + use_grpc=use_grpc ) self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace) self.nb_threads = nb_threads diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index c934c95a..c698a55f 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -22,23 +22,23 @@ THE SOFTWARE. """ +import http.client as http_client import logging import os import sys import time +import uuid from json.decoder import JSONDecodeError + import requests -import uuid -import http.client as http_client import urllib3 - from pypac import PACSession from pypac.parser import PACFile from urllib3.exceptions import InsecureRequestWarning -from .scanossbase import ScanossBase from . import __version__ - +from .constants import DEFAULT_TIMEOUT, MIN_TIMEOUT +from .scanossbase import ScanossBase DEFAULT_URL = 'https://api.osskb.org/scan/direct' # default free service URL DEFAULT_URL2 = 'https://api.scanoss.com/scan/direct' # default premium service URL @@ -52,7 +52,7 @@ class ScanossApi(ScanossBase): Currently support posting scan requests to the SCANOSS streaming API """ - def __init__( # noqa: PLR0913, PLR0915 + def __init__( # noqa: PLR0912, PLR0913, PLR0915 self, scan_format: str = None, flags: str = None, @@ -61,7 +61,7 @@ def __init__( # noqa: PLR0913, PLR0915 debug: bool = False, trace: bool = False, quiet: bool = False, - timeout: int = 180, + timeout: int = DEFAULT_TIMEOUT, ver_details: str = None, ignore_cert_errors: bool = False, proxy: str = None, @@ -87,30 +87,28 @@ def __init__( # noqa: PLR0913, PLR0915 HTTPS_PROXY='http://:' """ super().__init__(debug, trace, quiet) - self.url = url - self.api_key = api_key self.sbom = None self.scan_format = scan_format if scan_format else 'plain' self.flags = flags - self.timeout = timeout if timeout > 5 else 180 + self.timeout = timeout if timeout > MIN_TIMEOUT else DEFAULT_TIMEOUT self.retry_limit = retry if retry >= 0 else 5 self.ignore_cert_errors = ignore_cert_errors self.req_headers = req_headers if req_headers else {} self.headers = {} - + # Set the correct URL/API key combination + self.url = url if url else SCANOSS_SCAN_URL + self.api_key = api_key if api_key else SCANOSS_API_KEY + if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'): + self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium if ver_details: self.headers['x-scanoss-client'] = ver_details if self.api_key: self.headers['X-Session'] = self.api_key self.headers['x-api-key'] = self.api_key - self.headers['User-Agent'] = f'scanoss-py/{__version__}' - self.headers['user-agent'] = f'scanoss-py/{__version__}' - self.load_generic_headers() - - self.url = url if url else SCANOSS_SCAN_URL - self.api_key = api_key if api_key else SCANOSS_API_KEY - if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'): - self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + user_agent = f'scanoss-py/{__version__}' + self.headers['User-Agent'] = user_agent + self.headers['user-agent'] = user_agent + self.load_generic_headers(url) if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) @@ -133,7 +131,7 @@ def __init__( # noqa: PLR0913, PLR0915 if self.proxies: self.session.proxies = self.proxies - def scan(self, wfp: str, context: str = None, scan_id: int = None): + def scan(self, wfp: str, context: str = None, scan_id: int = None): # noqa: PLR0912, PLR0915 """ Scan the specified WFP and return the JSON object :param wfp: WFP to scan @@ -192,7 +190,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): else: self.print_stderr(f'Warning: No response received from {self.url}. Retrying...') time.sleep(5) - elif r.status_code == 503: # Service limits have most likely been reached + elif r.status_code == requests.codes.service_unavailable: # Service limits most likely reached self.print_stderr( f'ERROR: SCANOSS API rejected the scan request ({request_id}) due to ' f'service limits being exceeded' @@ -202,7 +200,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): f'ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected ' f'for {self.url} due to service limits being exceeded.' ) - elif r.status_code >= 400: + elif r.status_code >= requests.codes.bad_request: if retry > self.retry_limit: # No response retry_limit or more times, fail self.save_bad_req_wfp(scan_files, request_id, scan_id) raise Exception( @@ -269,7 +267,7 @@ def set_sbom(self, sbom): self.sbom = sbom return self - def load_generic_headers(self): + def load_generic_headers(self, url): """ Adds custom headers from req_headers to the headers collection. @@ -279,7 +277,7 @@ def load_generic_headers(self): if self.req_headers: # Load generic headers for key, value in self.req_headers.items(): if key == 'x-api-key': # Set premium URL if x-api-key header is set - if not self.url and not os.environ.get('SCANOSS_SCAN_URL'): + if not url and not os.environ.get('SCANOSS_SCAN_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.api_key = value self.headers[key] = value diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 4c11f715..fc3d3be3 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -23,8 +23,12 @@ """ import concurrent.futures +import http.client as http_client import json +import logging import os +import sys +import time import uuid from dataclasses import dataclass from enum import IntEnum @@ -32,15 +36,20 @@ from urllib.parse import urlparse import grpc +import requests +import urllib3 from google.protobuf.json_format import MessageToDict, ParseDict +from pypac import PACSession from pypac.parser import PACFile from pypac.resolver import ProxyResolver +from urllib3.exceptions import InsecureRequestWarning from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub from scanoss.constants import DEFAULT_TIMEOUT from . import __version__ from .api.common.v2.scanoss_common_pb2 import ( + ComponentsRequest, EchoRequest, EchoResponse, PurlRequest, @@ -62,7 +71,7 @@ from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub -from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import VulnerabilityResponse +from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import ComponentsVulnerabilityResponse from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub from .scanossbase import ScanossBase @@ -70,21 +79,20 @@ DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service URL SCANOSS_GRPC_URL = os.environ.get('SCANOSS_GRPC_URL') if os.environ.get('SCANOSS_GRPC_URL') else DEFAULT_URL SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else '' +DEFAULT_URI_PREFIX = '/v2' -MAX_CONCURRENT_REQUESTS = 5 +MAX_CONCURRENT_REQUESTS = 5 # Maximum number of concurrent requests to make class ScanossGrpcError(Exception): """ Custom exception for SCANOSS gRPC errors """ - pass class ScanossGrpcStatusCode(IntEnum): """Status codes for SCANOSS gRPC responses""" - SUCCESS = 1 SUCCESS_WITH_WARNINGS = 2 FAILED_WITH_WARNINGS = 3 @@ -96,7 +104,7 @@ class ScanossGrpc(ScanossBase): Client for gRPC functionality """ - def __init__( # noqa: PLR0913, PLR0915 + def __init__( # noqa: PLR0912, PLR0913, PLR0915 self, url: str = None, debug: bool = False, @@ -110,6 +118,8 @@ def __init__( # noqa: PLR0913, PLR0915 grpc_proxy: str = None, pac: PACFile = None, req_headers: dict = None, + ignore_cert_errors: bool = False, + use_grpc: bool = False, ): """ @@ -127,27 +137,55 @@ def __init__( # noqa: PLR0913, PLR0915 grpc_proxy='http://:' """ super().__init__(debug, trace, quiet) - self.url = url self.api_key = api_key if api_key else SCANOSS_API_KEY self.timeout = timeout self.proxy = proxy self.grpc_proxy = grpc_proxy self.pac = pac - self.req_headers = req_headers self.metadata = [] + self.ignore_cert_errors = ignore_cert_errors + self.use_grpc = use_grpc + self.req_headers = req_headers if req_headers else {} + self.headers = {} + self.retry_limit = 2 # default retry limit if self.api_key: self.metadata.append(('x-api-key', api_key)) # Set API key if we have one + self.headers['X-Session'] = self.api_key + self.headers['x-api-key'] = self.api_key if ver_details: self.metadata.append(('x-scanoss-client', ver_details)) - self.metadata.append(('user-agent', f'scanoss-py/{__version__}')) - self.load_generic_headers() - + self.headers['x-scanoss-client'] = ver_details + user_agent = f'scanoss-py/{__version__}' + self.metadata.append(('user-agent', user_agent)) + self.headers['User-Agent'] = user_agent + self.headers['user-agent'] = user_agent + self.headers['Content-Type'] = 'application/json' + # Set the correct URL/API key combination self.url = url if url else SCANOSS_GRPC_URL if self.api_key and not url and not os.environ.get('SCANOSS_GRPC_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + self.load_generic_headers(url) self.url = self.url.lower() - self.orig_url = self.url # Used for proxy lookup + self.orig_url = self.url.strip().rstrip('/') # Used for proxy lookup + # REST setup + if self.trace: + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + http_client.HTTPConnection.debuglevel = 1 + if pac and not proxy: # Set up a PAC session if requested (and no proxy has been explicitly set) + self.print_debug('Setting up PAC session...') + self.session = PACSession(pac=pac) + else: + self.session = requests.sessions.Session() + if self.ignore_cert_errors: + self.print_debug('Ignoring cert errors...') + urllib3.disable_warnings(InsecureRequestWarning) + self.session.verify = False + elif ca_cert: + self.session.verify = ca_cert + self.proxies = {'https': proxy, 'http': proxy} if proxy else None + if self.proxies: + self.session.proxies = self.proxies secure = True if self.url.startswith('https:') else False # Is it a secure connection? if self.url.startswith('http'): @@ -162,7 +200,7 @@ def __init__( # noqa: PLR0913, PLR0915 cert_data = ScanossGrpc._load_cert(ca_cert) self.print_debug(f'Setting up (secure: {secure}) connection to {self.url}...') self._get_proxy_config() - if secure is False: # insecure connection + if not secure: # insecure connection self.comp_search_stub = ComponentsStub(grpc.insecure_channel(self.url)) self.crypto_stub = CryptographyStub(grpc.insecure_channel(self.url)) self.dependencies_stub = DependenciesStub(grpc.insecure_channel(self.url)) @@ -206,17 +244,6 @@ def deps_echo(self, message: str = 'Hello there!') -> str: f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' ) else: - # self.print_stderr(f'resp: {resp} - call: {call}') - # response_id = "" - # if not call: - # self.print_stderr(f'No call to leverage.') - # for key, value in call.trailing_metadata(): - # print('Greeter client received trailing metadata: key=%s value=%s' % (key, value)) - # - # for key, value in call.trailing_metadata(): - # if key == 'x-response-id': - # response_id = value - # self.print_stderr(f'Response ID: {response_id}. Metadata: {call.trailing_metadata()}') if resp: return resp.message self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}') @@ -264,54 +291,70 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: if not dependencies: self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None - files_json = dependencies.get('files') - if files_json is None or len(files_json) == 0: - self.print_stderr('ERROR: No dependency data supplied to send to gRPC service.') + self.print_stderr('ERROR: No dependency data supplied to send to decoration service.') return None - - def process_file(file): - request_id = str(uuid.uuid4()) - try: - file_request = {'files': [file]} - - request = ParseDict(file_request, DependencyRequest()) - request.depth = depth - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) - self.print_debug(f'Sending dependency data for decoration (rqId: {request_id})...') - resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout) - - return MessageToDict(resp, preserving_proto_field_name=True) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - return None - all_responses = [] + # determine if we are using gRPC or REST based on the use_grpc flag + process_file = self._process_dep_file_grpc if self.use_grpc else self._process_dep_file_rest + # Process the dependency files in parallel with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS) as executor: - future_to_file = {executor.submit(process_file, file): file for file in files_json} - + future_to_file = {executor.submit(process_file, file, depth): file for file in files_json} for future in concurrent.futures.as_completed(future_to_file): response = future.result() if response: all_responses.append(response) - - SUCCESS_STATUS = 'SUCCESS' - - merged_response = {'files': [], 'status': {'status': SUCCESS_STATUS, 'message': 'Success'}} + # End of concurrent processing + success_status = 'SUCCESS' + merged_response = {'files': [], 'status': {'status': success_status, 'message': 'Success'}} + # Merge the responses for response in all_responses: if response: if 'files' in response and len(response['files']) > 0: merged_response['files'].append(response['files'][0]) - # Overwrite the status if the any of the responses was not successful - if 'status' in response and response['status']['status'] != SUCCESS_STATUS: + # Overwrite the status if any of the responses was not successful + if 'status' in response and response['status']['status'] != success_status: merged_response['status'] = response['status'] return merged_response + def _process_dep_file_grpc(self, file, depth: int = 1) -> dict: + """ + Process a single file using gRPC + + :param file: dependency file purls + :param depth: depth to search (default: 1) + :return: response JSON or None + """ + request_id = str(uuid.uuid4()) + try: + file_request = {'files': [file]} + request = ParseDict(file_request, DependencyRequest()) + request.depth = depth + metadata = self.metadata[:] + metadata.append(('x-request-id', request_id)) + self.print_debug(f'Sending dependency data via gRPC for decoration (rqId: {request_id})...') + resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout) + return MessageToDict(resp, preserving_proto_field_name=True) + except Exception as e: + self.print_stderr( + f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' + ) + return None + def get_vulnerabilities_json(self, purls: dict) -> dict: + """ + Client function to call the rpc for Vulnerability GetVulnerabilities + It will either use REST (default) or gRPC depending on the use_grpc flag + :param purls: Message to send to the service + :return: Server response or None + """ + if self.use_grpc: + return self._get_vulnerabilities_grpc(purls) + else: + return self._get_vulnerabilities_rest(purls) + + def _get_vulnerabilities_grpc(self, purls: dict) -> dict: """ Client function to call the rpc for Vulnerability GetVulnerabilities :param purls: Message to send to the service @@ -321,13 +364,13 @@ def get_vulnerabilities_json(self, purls: dict) -> dict: self.print_stderr('ERROR: No message supplied to send to gRPC service.') return None request_id = str(uuid.uuid4()) - resp: VulnerabilityResponse + resp: ComponentsVulnerabilityResponse try: - request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object + request = ParseDict(purls, ComponentsRequest()) # Parse the JSON/Dict into the purl request object metadata = self.metadata[:] metadata.append(('x-request-id', request_id)) # Set a Request ID self.print_debug(f'Sending vulnerability data for decoration (rqId: {request_id})...') - resp = self.vuln_stub.GetVulnerabilities(request, metadata=metadata, timeout=self.timeout) + resp = self.vuln_stub.GetComponentsVulnerabilities(request, metadata=metadata, timeout=self.timeout) except Exception as e: self.print_stderr( f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' @@ -462,14 +505,11 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional dict: The parsed gRPC response as a dictionary, or None if something went wrong """ request_id = str(uuid.uuid4()) - if isinstance(request_input, dict): request_obj = ParseDict(request_input, request_type()) else: request_obj = request_input - metadata = self.metadata[:] + [('x-request-id', request_id)] - self.print_debug(debug_msg.format(rqId=request_id)) try: resp = rpc_method(request_obj, metadata=metadata, timeout=self.timeout) @@ -477,7 +517,6 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional raise ScanossGrpcError( f'{e.__class__.__name__} while sending gRPC message (rqId: {request_id}): {e.details()}' ) - if resp and not self._check_status_response(resp.status, request_id): return None @@ -661,7 +700,7 @@ def get_versions_in_range_for_purl(self, request: Dict) -> Optional[Dict]: 'Sending data for cryptographic versions in range decoration (rqId: {rqId})...', ) - def load_generic_headers(self): + def load_generic_headers(self, url): """ Adds custom headers from req_headers to metadata. @@ -671,17 +710,106 @@ def load_generic_headers(self): if self.req_headers: # Load generic headers for key, value in self.req_headers.items(): if key == 'x-api-key': # Set premium URL if x-api-key header is set - if not self.url and not os.environ.get('SCANOSS_GRPC_URL'): + if not url and not os.environ.get('SCANOSS_GRPC_URL'): self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium self.api_key = value self.metadata.append((key, value)) + self.headers[key] = value + + # + # End of gRPC Client Functions + # + # Start of REST Client Functions + # + def rest_post(self, uri: str, request_id: str, data: dict) -> dict: + """ + Post the specified data to the given URI. + :param request_id: request id + :param uri: URI to post to + :param data: json data to post + :return: JSON response or None + """ + if not uri: + self.print_stderr('Error: Missing URI. Cannot search for project.') + return None + self.print_trace(f'Sending REST POST data to {uri}...') + headers = self.headers + headers['x-request-id'] = request_id # send a unique request id for each post + retry = 0 # Add some retry logic to cater for timeouts, etc. + while retry <= self.retry_limit: + retry += 1 + try: + response = self.session.post(uri, headers=headers, json=data, timeout=self.timeout) + response.raise_for_status() # Raises an HTTPError for bad responses + return response.json() + except requests.exceptions.RequestException as e: + self.print_stderr(f'Error: Problem posting data to {uri}: {e}') + except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e: + self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') + raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: + if retry > self.retry_limit: # Timed out retry_limit or more times, fail + self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing decoration data ({request_id}): {e}') + raise Exception( + f'ERROR: The SCANOSS Decoration API request timed out ({e.__class__.__name__}) for {uri}' + ) from e + else: + self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...') + time.sleep(5) + except Exception as e: + self.print_stderr( + f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) to {uri}: {e}' + ) + raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e + return None + def _get_vulnerabilities_rest(self, purls: dict): + """ + Get the vulnerabilities for the given purls using REST API + :param purls: Purl Request dictionary + :return: Vulnerability Response, or None if the request was unsuccessful + """ + if not purls: + self.print_stderr('ERROR: No message supplied to send to REST decoration service.') + return None + request_id = str(uuid.uuid4()) + self.print_debug(f'Sending data for Vulnerabilities via REST (request id: {request_id})...') + response = self.rest_post(f'{self.orig_url}{DEFAULT_URI_PREFIX}/vulnerabilities/components', request_id, purls) + self.print_trace(f'Received response for Vulnerabilities via REST (request id: {request_id}): {response}') + if response: + # Parse the JSON/Dict into the purl response + resp_obj = ParseDict(response, ComponentsVulnerabilityResponse(), True) + if resp_obj: + self.print_debug(f'Vulnerability Response: {resp_obj}') + if not self._check_status_response(resp_obj.status, request_id): + return None + del response['status'] + return response + return None + + def _process_dep_file_rest(self, file, depth: int = 1) -> dict: + """ + Porcess a single dependency file using REST + + :param file: dependency file purls + :param depth: depth to search (default: 1) + :return: response JSON or None + """ + request_id = str(uuid.uuid4()) + self.print_debug(f'Sending data for Dependencies via REST (request id: {request_id})...') + file_request = {'files': [file], 'depth': depth} + response = self.rest_post(f'{self.orig_url}{DEFAULT_URI_PREFIX}/dependencies/dependencies', + request_id, file_request + ) + self.print_trace(f'Received response for Dependencies via REST (request id: {request_id}): {response}') + if response: + return response + return None # # End of ScanossGrpc Class # - @dataclass class GrpcConfig: url: str = DEFAULT_URL @@ -711,3 +839,7 @@ def create_grpc_config_from_args(args) -> GrpcConfig: proxy=getattr(args, 'proxy', None), grpc_proxy=getattr(args, 'grpc_proxy', None), ) + +# +# End of GrpcConfig class +# \ No newline at end of file From 2502ce58cc83ebea6ebcdb3ceec8daedd25c1bdd Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:39:58 +0200 Subject: [PATCH 390/489] [SP-2874] feat: add licenses sub-command, add support for ingesting CDX, add CDX input validation (#131) * [SP-2874] feat: add licenses sub-command, add support for ingesting CDX, add CDX input validation * [SP-2874] feat: support CDX input file in crypto decoration commands * [SP-2874] chore: update dependency versions, refactor to reduce method complexity * [SP-2874] chore: use python 3.9 in all workflows * [SP-2874] chore: update protobuf * [SP-2874] chore: update all remaining protobufs * [SP-2874] chore: bump version * [SP-2874] chore: update changelog, documentation and dockerfile * [SP-2874] chore: update scanoss.json * [SP-2874] chore: update scanoss.json * [SP-2874] chore: update changelog and version * [SP-2991] fix: update to papi latest definitions * [SP-2874] chore: update version and changelog * [SP-2874] fix: adapt for new components request * [SP-2874] feat: add REST support for licenses endpoint * [SP-2874] chore: update workflow python version * [SP-2874] fix: scancode dockerfile execution * [SP-2874] chore: update pkg requirements * [SP-2874] chore: fix click version as workaround for scancode-toolkit-mini * [SP-2874] chore: add api key on github workflows * [SP-2874] chore: add api key on github workflows * add api key * env cleanup --------- Co-authored-by: eeisegn --- .github/workflows/container-local-test.yml | 9 +- .github/workflows/container-publish-ghcr.yml | 4 +- .github/workflows/python-local-test.yml | 5 + .github/workflows/python-publish-pypi.yml | 7 +- .github/workflows/python-publish-testpypi.yml | 7 +- .github/workflows/version-tag.yml | 2 +- CHANGELOG.md | 21 +- CLIENT_HELP.md | 38 + Dockerfile | 3 +- requirements.txt | 6 +- scanoss.json | 12 +- setup.cfg | 5 +- .../options/annotations_pb2.py | 30 +- .../options/annotations_pb2.pyi | 48 + .../options/annotations_pb2_grpc.py | 20 + .../options/openapiv2_pb2.py | 209 +-- .../options/openapiv2_pb2.pyi | 1317 +++++++++++++++++ .../options/openapiv2_pb2_grpc.py | 20 + src/scanoss/__init__.py | 2 +- .../api/common/v2/scanoss_common_pb2_grpc.py | 6 +- src/scanoss/cli.py | 318 ++-- src/scanoss/components.py | 59 +- src/scanoss/cryptography.py | 56 +- src/scanoss/cyclonedx.py | 22 + src/scanoss/scanossgrpc.py | 84 +- tests/data/requirements.txt | 4 +- 26 files changed, 1988 insertions(+), 326 deletions(-) create mode 100644 src/protoc_gen_swagger/options/annotations_pb2.pyi create mode 100644 src/protoc_gen_swagger/options/openapiv2_pb2.pyi diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 6021e2c0..447e6053 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -5,10 +5,10 @@ on: workflow_dispatch: push: branches: - - 'main' + - "main" pull_request: branches: - - 'main' + - "main" env: IMAGE_BASE: scanoss/scanoss-py-base @@ -27,7 +27,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9.x' + python-version: "3.9.x" - name: Install Dependencies run: | @@ -91,10 +91,11 @@ jobs: docker image ls -a docker run ${{ env.IMAGE_NAME }} version docker run ${{ env.IMAGE_NAME }} utils fast - docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests + docker run -e SCANOSS_API_KEY="${{ secrets.SC_API_KEY }}" -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then echo "Error: Scan test did not produce any results. Failing" exit 1 fi + diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 14125a6e..3d8fb39d 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -30,7 +30,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9.x" + python-version: '3.9.x' - name: Install Dependencies run: | @@ -130,7 +130,7 @@ jobs: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} version docker run ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} utils fast - docker run -v "$(pwd)":"/scanoss" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} scan -o results.json tests + docker run -e SCANOSS_API_KEY="${{ secrets.SC_API_KEY }}" -v "$(pwd)":"/scanoss" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} scan -o results.json tests id_count=$(cat results.json | grep '"id":' | wc -l) echo "ID Count: $id_count" if [[ $id_count -lt 1 ]]; then diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 24f36026..50e13960 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -13,6 +13,9 @@ on: permissions: contents: read +env: + SCANOSS_API_KEY: ${{ secrets.SC_API_KEY }} + jobs: build: runs-on: ubuntu-latest @@ -71,6 +74,7 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + - name: Run Tests HPSM (fast winnowing) run: | @@ -85,6 +89,7 @@ jobs: echo "Error: WFP test did not produce any results. Failing" exit 1 fi + - name: Run Unit Tests run: | diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 793692c3..2838bed9 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -7,6 +7,9 @@ on: tags: - "v*.*.*" +env: + SCANOSS_API_KEY: ${{ secrets.SC_API_KEY }} + jobs: deploy: runs-on: ubuntu-latest @@ -16,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9.x" + python-version: '3.9.x' - name: Install dependencies run: | @@ -70,7 +73,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9.x" + python-version: '3.9.x' - name: Install Remote Package uses: nick-fields/retry@v3 diff --git a/.github/workflows/python-publish-testpypi.yml b/.github/workflows/python-publish-testpypi.yml index e0ecfa1c..d44c02a0 100644 --- a/.github/workflows/python-publish-testpypi.yml +++ b/.github/workflows/python-publish-testpypi.yml @@ -6,6 +6,9 @@ on: [workflow_dispatch] permissions: contents: read +env: + SCANOSS_API_KEY: ${{ secrets.SC_API_KEY }} + jobs: deploy: runs-on: ubuntu-latest @@ -15,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9.x" + python-version: '3.9.x' - name: Install Dependencies run: | @@ -65,7 +68,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9.x" + python-version: '3.9.x' - name: Install Remote Package run: | diff --git a/.github/workflows/version-tag.yml b/.github/workflows/version-tag.yml index 0d333050..55afecb0 100644 --- a/.github/workflows/version-tag.yml +++ b/.github/workflows/version-tag.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9.x" + python-version: '3.9.x' - name: Determine Tag id: taggerVersion run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3d1d38..663c4eb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.33.0] - 2025-09-19 +### Added +- Add `licenses` sub-command to `component` command +- Add support for ingesting CDX to all decoration commands +- Add CDX input validation + ## [1.32.0] - 2025-09-01 ### Added - Switched vulnerability and dependency APIs to use REST by default @@ -176,7 +182,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.20.2] - 2025-02-26 ### Fixed -- Fixed provenance command +- Fixed provenance command ## [1.20.1] - 2025-02-18 ### Added @@ -238,7 +244,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.17.5] - 2024-11-12 ### Fixed - Fix dependencies scan result structure - + ## [1.17.4] - 2024-11-08 ### Fixed - Fix backslashes in file paths on Windows @@ -255,7 +261,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added supplier to SPDX packages ### Changed -- Changed undeclared summary output +- Changed undeclared summary output ## [1.17.1] - 2024-10-24 ### Fixed @@ -288,7 +294,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added support for Python3.12 - Module `pkg_resources` has been replaced with `importlib_resources` -- Added support for UTF-16 filenames +- Added support for UTF-16 filenames ## [1.13.0] - 2024-06-05 ### Added @@ -367,11 +373,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.6.0] - 2023-06-16 ### Added - Added support for High Precision Snippet Matching (`--hpsm` or `-H`) while scanning - - `scanoss-py scan --hpsm ...` + - `scanoss-py scan --hpsm ...` ## [1.5.2] - 2023-06-13 ### Added -- Added retry limit option (`--retry`) while scanning +- Added retry limit option (`--retry`) while scanning - `--retry 0` will fail immediately ## [1.5.1] - 2023-04-21 @@ -660,4 +666,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.31.3]: https://github.com/scanoss/scanoss.py/compare/v1.31.2...v1.31.3 [1.31.4]: https://github.com/scanoss/scanoss.py/compare/v1.31.3...v1.31.4 [1.31.5]: https://github.com/scanoss/scanoss.py/compare/v1.31.4...v1.31.5 -[1.31.5]: https://github.com/scanoss/scanoss.py/compare/v1.31.5...v1.32.0 +[1.32.0]: https://github.com/scanoss/scanoss.py/compare/v1.31.5...v1.32.0 +[1.33.0]: https://github.com/scanoss/scanoss.py/compare/v1.32.0...v1.33.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index e6f3e865..5707adbc 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -337,6 +337,44 @@ The following command provides the capability to search the SCANOSS KB for compo scanoss-py comp prov -p "pkg:github/unoconv/unoconv" --origin ``` +#### Component Licenses +The following command provides the capability to search the SCANOSS KB for licenses for Open Source components: +```bash +scanoss-py comp licenses -p "pkg:github/jquery/jquery" -p "pkg:npm/express" +``` +It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): +```bash +scanoss-py comp licenses -i purl-input.json -o component-licenses.json +``` + +The licenses command also supports CycloneDX (CDX) input files. You can provide a CycloneDX SBOM file and retrieve license information for all components: +```bash +scanoss-py comp licenses -i cyclonedx-sbom.json -o component-licenses.json +``` + +### CDX Input Support for Component Commands +Several component commands now support CycloneDX (CDX) input files. This allows you to analyze components from existing SBOM files: + +**Supported commands with CDX input:** +- `comp vulns` - Analyze vulnerabilities from CDX file +- `comp licenses` - Retrieve licenses from CDX file +- `comp crypto` - Detect cryptographic algorithms from CDX file +- `comp semgrep` - Find semgrep issues from CDX file + +**Example using CDX input:** +```bash +# Analyze vulnerabilities from a CycloneDX SBOM +scanoss-py comp vulns -i sbom.cdx.json -o vulnerabilities.json + +# Get licenses for all components in a CycloneDX SBOM +scanoss-py comp licenses -i sbom.cdx.json -o licenses.json + +# Detect cryptographic usage from CDX +scanoss-py comp crypto -i sbom.cdx.json -o crypto-findings.json +``` + +The CDX input file is automatically validated to ensure it's a valid CycloneDX format before processing. + ### Results Commands The `results` command provides the capability to operate on scan results. For example: diff --git a/Dockerfile b/Dockerfile index f1fb4648..2005bc3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN pip3 install --no-cache-dir /install/scanoss-*-py3-none-any.whl RUN pip3 install --no-cache-dir scanoss_winnowing RUN pip3 install --no-cache-dir -r /install/requirements-dev.txt RUN pip3 install --no-cache-dir scancode-toolkit-mini +RUN pip3 install --no-cache-dir click==8.2.1 # Temporary workaround for scancode-toolkit-mini (https://github.com/aboutcode-org/scancode-toolkit/issues/4573) # Download compile and install typecode-libmagic from source (as there is not ARM wheel available) ADD https://github.com/nexB/typecode_libmagic_from_sources/archive/refs/tags/v5.39.210212.tar.gz /install/ @@ -66,7 +67,7 @@ RUN curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | # Setup working directory and user WORKDIR /scanoss # Run scancode once to setup any initial files, etc. so that it'll run faster later -RUN scancode -p --only-findings --quiet --json /scanoss/scancode-dependencies.json /scanoss && rm -f /scanoss/scancode-dependencies.json +RUN scancode --package --only-findings --quiet --json /scanoss/scancode-dependencies.json /scanoss && rm -f /scanoss/scancode-dependencies.json # Image with no default entry point FROM no_entry_point AS jenkins diff --git a/requirements.txt b/requirements.txt index 1d3fc07c..9beb5da0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,9 @@ requests crc32c>=2.2 binaryornot progress -grpcio>1.42.0 -protobuf>3.19.1 +grpcio>=1.73.1 +protobuf>=6.3.1 +protoc-gen-openapiv2 pypac urllib3 pyOpenSSL @@ -13,5 +14,4 @@ packageurl-python pathspec jsonschema crc -protoc-gen-openapiv2 cyclonedx-python-lib[validation] \ No newline at end of file diff --git a/scanoss.json b/scanoss.json index 813b121a..954cd89a 100644 --- a/scanoss.json +++ b/scanoss.json @@ -3,9 +3,9 @@ "skip": { "patterns": { "scanning": [ - "src/protoc_gen_swagger/", - "src/scanoss/api/", - "docs/make.bat" + "src/protoc_gen_swagger", + "docs", + "scanoss_common_pb2_grpc.py" ] }, "sizes": {} @@ -16,6 +16,8 @@ { "purl": "pkg:github/scanoss/scanoss.py" } - ] + ], + "remove": [] } -} \ No newline at end of file +} + diff --git a/setup.cfg b/setup.cfg index cb37a860..602b0b3e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,8 +29,9 @@ install_requires = crc32c>=2.2 binaryornot progress - grpcio>1.42.0 - protobuf>3.19.1 + grpcio>=1.73.1 + protobuf>=6.3.1 + protoc-gen-openapiv2 pypac pyOpenSSL google-api-core diff --git a/src/protoc_gen_swagger/options/annotations_pb2.py b/src/protoc_gen_swagger/options/annotations_pb2.py index c568f388..d08ab680 100644 --- a/src/protoc_gen_swagger/options/annotations_pb2.py +++ b/src/protoc_gen_swagger/options/annotations_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: protoc-gen-swagger/options/annotations.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'protoc-gen-swagger/options/annotations.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -17,15 +28,10 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,protoc-gen-swagger/options/annotations.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a google/protobuf/descriptor.proto\x1a*protoc-gen-swagger/options/openapiv2.proto:j\n\x11openapiv2_swagger\x12\x1c.google.protobuf.FileOptions\x18\x92\x08 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Swagger:p\n\x13openapiv2_operation\x12\x1e.google.protobuf.MethodOptions\x18\x92\x08 \x01(\x0b\x32\x32.grpc.gateway.protoc_gen_swagger.options.Operation:k\n\x10openapiv2_schema\x12\x1f.google.protobuf.MessageOptions\x18\x92\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema:e\n\ropenapiv2_tag\x12\x1f.google.protobuf.ServiceOptions\x18\x92\x08 \x01(\x0b\x32,.grpc.gateway.protoc_gen_swagger.options.Tag:l\n\x0fopenapiv2_field\x12\x1d.google.protobuf.FieldOptions\x18\x92\x08 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchemaBCZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.annotations_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - google_dot_protobuf_dot_descriptor__pb2.FileOptions.RegisterExtension(openapiv2_swagger) - google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(openapiv2_operation) - google_dot_protobuf_dot_descriptor__pb2.MessageOptions.RegisterExtension(openapiv2_schema) - google_dot_protobuf_dot_descriptor__pb2.ServiceOptions.RegisterExtension(openapiv2_tag) - google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(openapiv2_field) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.annotations_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' # @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/annotations_pb2.pyi b/src/protoc_gen_swagger/options/annotations_pb2.pyi new file mode 100644 index 00000000..82a4bd56 --- /dev/null +++ b/src/protoc_gen_swagger/options/annotations_pb2.pyi @@ -0,0 +1,48 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import google.protobuf.descriptor +import google.protobuf.descriptor_pb2 +import google.protobuf.internal.extension_dict +import protoc_gen_swagger.options.openapiv2_pb2 + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +OPENAPIV2_SWAGGER_FIELD_NUMBER: builtins.int +OPENAPIV2_OPERATION_FIELD_NUMBER: builtins.int +OPENAPIV2_SCHEMA_FIELD_NUMBER: builtins.int +OPENAPIV2_TAG_FIELD_NUMBER: builtins.int +OPENAPIV2_FIELD_FIELD_NUMBER: builtins.int +openapiv2_swagger: google.protobuf.internal.extension_dict._ExtensionFieldDescriptor[google.protobuf.descriptor_pb2.FileOptions, protoc_gen_swagger.options.openapiv2_pb2.Swagger] +"""ID assigned by protobuf-global-extension-registry@google.com for grpc-gateway project. + +All IDs are the same, as assigned. It is okay that they are the same, as they extend +different descriptor messages. +""" +openapiv2_operation: google.protobuf.internal.extension_dict._ExtensionFieldDescriptor[google.protobuf.descriptor_pb2.MethodOptions, protoc_gen_swagger.options.openapiv2_pb2.Operation] +"""ID assigned by protobuf-global-extension-registry@google.com for grpc-gateway project. + +All IDs are the same, as assigned. It is okay that they are the same, as they extend +different descriptor messages. +""" +openapiv2_schema: google.protobuf.internal.extension_dict._ExtensionFieldDescriptor[google.protobuf.descriptor_pb2.MessageOptions, protoc_gen_swagger.options.openapiv2_pb2.Schema] +"""ID assigned by protobuf-global-extension-registry@google.com for grpc-gateway project. + +All IDs are the same, as assigned. It is okay that they are the same, as they extend +different descriptor messages. +""" +openapiv2_tag: google.protobuf.internal.extension_dict._ExtensionFieldDescriptor[google.protobuf.descriptor_pb2.ServiceOptions, protoc_gen_swagger.options.openapiv2_pb2.Tag] +"""ID assigned by protobuf-global-extension-registry@google.com for grpc-gateway project. + +All IDs are the same, as assigned. It is okay that they are the same, as they extend +different descriptor messages. +""" +openapiv2_field: google.protobuf.internal.extension_dict._ExtensionFieldDescriptor[google.protobuf.descriptor_pb2.FieldOptions, protoc_gen_swagger.options.openapiv2_pb2.JSONSchema] +"""ID assigned by protobuf-global-extension-registry@google.com for grpc-gateway project. + +All IDs are the same, as assigned. It is okay that they are the same, as they extend +different descriptor messages. +""" diff --git a/src/protoc_gen_swagger/options/annotations_pb2_grpc.py b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py index 2daafffe..d1e1d60e 100644 --- a/src/protoc_gen_swagger/options/annotations_pb2_grpc.py +++ b/src/protoc_gen_swagger/options/annotations_pb2_grpc.py @@ -1,4 +1,24 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings + +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in protoc_gen_swagger/options/annotations_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2.py b/src/protoc_gen_swagger/options/openapiv2_pb2.py index 0df96e43..b0db8ffe 100644 --- a/src/protoc_gen_swagger/options/openapiv2_pb2.py +++ b/src/protoc_gen_swagger/options/openapiv2_pb2.py @@ -1,11 +1,22 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE # source: protoc-gen-swagger/options/openapiv2.proto +# Protobuf Python Version: 6.31.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 0, + '', + 'protoc-gen-swagger/options/openapiv2.proto' +) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -17,102 +28,102 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*protoc-gen-swagger/options/openapiv2.proto\x12\'grpc.gateway.protoc_gen_swagger.options\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xa0\x07\n\x07Swagger\x12\x0f\n\x07swagger\x18\x01 \x01(\t\x12;\n\x04info\x18\x02 \x01(\x0b\x32-.grpc.gateway.protoc_gen_swagger.options.Info\x12\x0c\n\x04host\x18\x03 \x01(\t\x12\x11\n\tbase_path\x18\x04 \x01(\t\x12O\n\x07schemes\x18\x05 \x03(\x0e\x32>.grpc.gateway.protoc_gen_swagger.options.Swagger.SwaggerScheme\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12R\n\tresponses\x18\n \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Swagger.ResponsesEntry\x12Z\n\x14security_definitions\x18\x0b \x01(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12U\n\rexternal_docs\x18\x0e \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12T\n\nextensions\x18\x0f \x03(\x0b\x32@.grpc.gateway.protoc_gen_swagger.options.Swagger.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"B\n\rSwaggerScheme\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04HTTP\x10\x01\x12\t\n\x05HTTPS\x10\x02\x12\x06\n\x02WS\x10\x03\x12\x07\n\x03WSS\x10\x04J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\r\x10\x0e\"\xa9\x05\n\tOperation\x12\x0c\n\x04tags\x18\x01 \x03(\t\x12\x0f\n\x07summary\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12U\n\rexternal_docs\x18\x04 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12\x14\n\x0coperation_id\x18\x05 \x01(\t\x12\x10\n\x08\x63onsumes\x18\x06 \x03(\t\x12\x10\n\x08produces\x18\x07 \x03(\t\x12T\n\tresponses\x18\t \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Operation.ResponsesEntry\x12\x0f\n\x07schemes\x18\n \x03(\t\x12\x12\n\ndeprecated\x18\x0b \x01(\x08\x12N\n\x08security\x18\x0c \x03(\x0b\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement\x12V\n\nextensions\x18\r \x03(\x0b\x32\x42.grpc.gateway.protoc_gen_swagger.options.Operation.ExtensionsEntry\x1a\x63\n\x0eResponsesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12@\n\x05value\x18\x02 \x01(\x0b\x32\x31.grpc.gateway.protoc_gen_swagger.options.Response:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01J\x04\x08\x08\x10\t\"\xab\x01\n\x06Header\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0e\n\x06\x66ormat\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x06 \x01(\t\x12\x0f\n\x07pattern\x18\r \x01(\tJ\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x07\x10\x08J\x04\x08\x08\x10\tJ\x04\x08\t\x10\nJ\x04\x08\n\x10\x0bJ\x04\x08\x0b\x10\x0cJ\x04\x08\x0c\x10\rJ\x04\x08\x0e\x10\x0fJ\x04\x08\x0f\x10\x10J\x04\x08\x10\x10\x11J\x04\x08\x11\x10\x12J\x04\x08\x12\x10\x13\"\xb8\x04\n\x08Response\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12?\n\x06schema\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Schema\x12O\n\x07headers\x18\x03 \x03(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.Response.HeadersEntry\x12Q\n\x08\x65xamples\x18\x04 \x03(\x0b\x32?.grpc.gateway.protoc_gen_swagger.options.Response.ExamplesEntry\x12U\n\nextensions\x18\x05 \x03(\x0b\x32\x41.grpc.gateway.protoc_gen_swagger.options.Response.ExtensionsEntry\x1a_\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12>\n\x05value\x18\x02 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Header:\x02\x38\x01\x1a/\n\rExamplesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"\xf9\x02\n\x04Info\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x18\n\x10terms_of_service\x18\x03 \x01(\t\x12\x41\n\x07\x63ontact\x18\x04 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.Contact\x12\x41\n\x07license\x18\x05 \x01(\x0b\x32\x30.grpc.gateway.protoc_gen_swagger.options.License\x12\x0f\n\x07version\x18\x06 \x01(\t\x12Q\n\nextensions\x18\x07 \x03(\x0b\x32=.grpc.gateway.protoc_gen_swagger.options.Info.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"3\n\x07\x43ontact\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\r\n\x05\x65mail\x18\x03 \x01(\t\"$\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"9\n\x15\x45xternalDocumentation\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\"\x9c\x02\n\x06Schema\x12H\n\x0bjson_schema\x18\x01 \x01(\x0b\x32\x33.grpc.gateway.protoc_gen_swagger.options.JSONSchema\x12\x15\n\rdiscriminator\x18\x02 \x01(\t\x12\x11\n\tread_only\x18\x03 \x01(\x08\x12U\n\rexternal_docs\x18\x05 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentation\x12)\n\x07\x65xample\x18\x06 \x01(\x0b\x32\x14.google.protobuf.AnyB\x02\x18\x01\x12\x16\n\x0e\x65xample_string\x18\x07 \x01(\tJ\x04\x08\x04\x10\x05\"\xe3\x05\n\nJSONSchema\x12\x0b\n\x03ref\x18\x03 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x65\x66\x61ult\x18\x07 \x01(\t\x12\x11\n\tread_only\x18\x08 \x01(\x08\x12\x0f\n\x07\x65xample\x18\t \x01(\t\x12\x13\n\x0bmultiple_of\x18\n \x01(\x01\x12\x0f\n\x07maximum\x18\x0b \x01(\x01\x12\x19\n\x11\x65xclusive_maximum\x18\x0c \x01(\x08\x12\x0f\n\x07minimum\x18\r \x01(\x01\x12\x19\n\x11\x65xclusive_minimum\x18\x0e \x01(\x08\x12\x12\n\nmax_length\x18\x0f \x01(\x04\x12\x12\n\nmin_length\x18\x10 \x01(\x04\x12\x0f\n\x07pattern\x18\x11 \x01(\t\x12\x11\n\tmax_items\x18\x14 \x01(\x04\x12\x11\n\tmin_items\x18\x15 \x01(\x04\x12\x14\n\x0cunique_items\x18\x16 \x01(\x08\x12\x16\n\x0emax_properties\x18\x18 \x01(\x04\x12\x16\n\x0emin_properties\x18\x19 \x01(\x04\x12\x10\n\x08required\x18\x1a \x03(\t\x12\r\n\x05\x61rray\x18\" \x03(\t\x12W\n\x04type\x18# \x03(\x0e\x32I.grpc.gateway.protoc_gen_swagger.options.JSONSchema.JSONSchemaSimpleTypes\x12\x0e\n\x06\x66ormat\x18$ \x01(\t\x12\x0c\n\x04\x65num\x18. \x03(\t\"w\n\x15JSONSchemaSimpleTypes\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41RRAY\x10\x01\x12\x0b\n\x07\x42OOLEAN\x10\x02\x12\x0b\n\x07INTEGER\x10\x03\x12\x08\n\x04NULL\x10\x04\x12\n\n\x06NUMBER\x10\x05\x12\n\n\x06OBJECT\x10\x06\x12\n\n\x06STRING\x10\x07J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13J\x04\x08\x13\x10\x14J\x04\x08\x17\x10\x18J\x04\x08\x1b\x10\x1cJ\x04\x08\x1c\x10\x1dJ\x04\x08\x1d\x10\x1eJ\x04\x08\x1e\x10\"J\x04\x08%\x10*J\x04\x08*\x10+J\x04\x08+\x10.\"w\n\x03Tag\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12U\n\rexternal_docs\x18\x03 \x01(\x0b\x32>.grpc.gateway.protoc_gen_swagger.options.ExternalDocumentationJ\x04\x08\x01\x10\x02\"\xdd\x01\n\x13SecurityDefinitions\x12\\\n\x08security\x18\x01 \x03(\x0b\x32J.grpc.gateway.protoc_gen_swagger.options.SecurityDefinitions.SecurityEntry\x1ah\n\rSecurityEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x46\n\x05value\x18\x02 \x01(\x0b\x32\x37.grpc.gateway.protoc_gen_swagger.options.SecurityScheme:\x02\x38\x01\"\x96\x06\n\x0eSecurityScheme\x12J\n\x04type\x18\x01 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Type\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x46\n\x02in\x18\x04 \x01(\x0e\x32:.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.In\x12J\n\x04\x66low\x18\x05 \x01(\x0e\x32<.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.Flow\x12\x19\n\x11\x61uthorization_url\x18\x06 \x01(\t\x12\x11\n\ttoken_url\x18\x07 \x01(\t\x12?\n\x06scopes\x18\x08 \x01(\x0b\x32/.grpc.gateway.protoc_gen_swagger.options.Scopes\x12[\n\nextensions\x18\t \x03(\x0b\x32G.grpc.gateway.protoc_gen_swagger.options.SecurityScheme.ExtensionsEntry\x1aI\n\x0f\x45xtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value:\x02\x38\x01\"K\n\x04Type\x12\x10\n\x0cTYPE_INVALID\x10\x00\x12\x0e\n\nTYPE_BASIC\x10\x01\x12\x10\n\x0cTYPE_API_KEY\x10\x02\x12\x0f\n\x0bTYPE_OAUTH2\x10\x03\"1\n\x02In\x12\x0e\n\nIN_INVALID\x10\x00\x12\x0c\n\x08IN_QUERY\x10\x01\x12\r\n\tIN_HEADER\x10\x02\"j\n\x04\x46low\x12\x10\n\x0c\x46LOW_INVALID\x10\x00\x12\x11\n\rFLOW_IMPLICIT\x10\x01\x12\x11\n\rFLOW_PASSWORD\x10\x02\x12\x14\n\x10\x46LOW_APPLICATION\x10\x03\x12\x14\n\x10\x46LOW_ACCESS_CODE\x10\x04\"\xc9\x02\n\x13SecurityRequirement\x12s\n\x14security_requirement\x18\x01 \x03(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementEntry\x1a)\n\x18SecurityRequirementValue\x12\r\n\x05scope\x18\x01 \x03(\t\x1a\x91\x01\n\x18SecurityRequirementEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x64\n\x05value\x18\x02 \x01(\x0b\x32U.grpc.gateway.protoc_gen_swagger.options.SecurityRequirement.SecurityRequirementValue:\x02\x38\x01\"\x81\x01\n\x06Scopes\x12I\n\x05scope\x18\x01 \x03(\x0b\x32:.grpc.gateway.protoc_gen_swagger.options.Scopes.ScopeEntry\x1a,\n\nScopeEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x43ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/optionsb\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.openapiv2_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' - _SWAGGER_RESPONSESENTRY._options = None - _SWAGGER_RESPONSESENTRY._serialized_options = b'8\001' - _SWAGGER_EXTENSIONSENTRY._options = None - _SWAGGER_EXTENSIONSENTRY._serialized_options = b'8\001' - _OPERATION_RESPONSESENTRY._options = None - _OPERATION_RESPONSESENTRY._serialized_options = b'8\001' - _OPERATION_EXTENSIONSENTRY._options = None - _OPERATION_EXTENSIONSENTRY._serialized_options = b'8\001' - _RESPONSE_HEADERSENTRY._options = None - _RESPONSE_HEADERSENTRY._serialized_options = b'8\001' - _RESPONSE_EXAMPLESENTRY._options = None - _RESPONSE_EXAMPLESENTRY._serialized_options = b'8\001' - _RESPONSE_EXTENSIONSENTRY._options = None - _RESPONSE_EXTENSIONSENTRY._serialized_options = b'8\001' - _INFO_EXTENSIONSENTRY._options = None - _INFO_EXTENSIONSENTRY._serialized_options = b'8\001' - _SCHEMA.fields_by_name['example']._options = None - _SCHEMA.fields_by_name['example']._serialized_options = b'\030\001' - _SECURITYDEFINITIONS_SECURITYENTRY._options = None - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_options = b'8\001' - _SECURITYSCHEME_EXTENSIONSENTRY._options = None - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_options = b'8\001' - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._options = None - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_options = b'8\001' - _SCOPES_SCOPEENTRY._options = None - _SCOPES_SCOPEENTRY._serialized_options = b'8\001' - _SWAGGER._serialized_start=145 - _SWAGGER._serialized_end=1073 - _SWAGGER_RESPONSESENTRY._serialized_start=813 - _SWAGGER_RESPONSESENTRY._serialized_end=912 - _SWAGGER_EXTENSIONSENTRY._serialized_start=914 - _SWAGGER_EXTENSIONSENTRY._serialized_end=987 - _SWAGGER_SWAGGERSCHEME._serialized_start=989 - _SWAGGER_SWAGGERSCHEME._serialized_end=1055 - _OPERATION._serialized_start=1076 - _OPERATION._serialized_end=1757 - _OPERATION_RESPONSESENTRY._serialized_start=813 - _OPERATION_RESPONSESENTRY._serialized_end=912 - _OPERATION_EXTENSIONSENTRY._serialized_start=914 - _OPERATION_EXTENSIONSENTRY._serialized_end=987 - _HEADER._serialized_start=1760 - _HEADER._serialized_end=1931 - _RESPONSE._serialized_start=1934 - _RESPONSE._serialized_end=2502 - _RESPONSE_HEADERSENTRY._serialized_start=2283 - _RESPONSE_HEADERSENTRY._serialized_end=2378 - _RESPONSE_EXAMPLESENTRY._serialized_start=2380 - _RESPONSE_EXAMPLESENTRY._serialized_end=2427 - _RESPONSE_EXTENSIONSENTRY._serialized_start=914 - _RESPONSE_EXTENSIONSENTRY._serialized_end=987 - _INFO._serialized_start=2505 - _INFO._serialized_end=2882 - _INFO_EXTENSIONSENTRY._serialized_start=914 - _INFO_EXTENSIONSENTRY._serialized_end=987 - _CONTACT._serialized_start=2884 - _CONTACT._serialized_end=2935 - _LICENSE._serialized_start=2937 - _LICENSE._serialized_end=2973 - _EXTERNALDOCUMENTATION._serialized_start=2975 - _EXTERNALDOCUMENTATION._serialized_end=3032 - _SCHEMA._serialized_start=3035 - _SCHEMA._serialized_end=3319 - _JSONSCHEMA._serialized_start=3322 - _JSONSCHEMA._serialized_end=4061 - _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_start=3864 - _JSONSCHEMA_JSONSCHEMASIMPLETYPES._serialized_end=3983 - _TAG._serialized_start=4063 - _TAG._serialized_end=4182 - _SECURITYDEFINITIONS._serialized_start=4185 - _SECURITYDEFINITIONS._serialized_end=4406 - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_start=4302 - _SECURITYDEFINITIONS_SECURITYENTRY._serialized_end=4406 - _SECURITYSCHEME._serialized_start=4409 - _SECURITYSCHEME._serialized_end=5199 - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_start=914 - _SECURITYSCHEME_EXTENSIONSENTRY._serialized_end=987 - _SECURITYSCHEME_TYPE._serialized_start=4965 - _SECURITYSCHEME_TYPE._serialized_end=5040 - _SECURITYSCHEME_IN._serialized_start=5042 - _SECURITYSCHEME_IN._serialized_end=5091 - _SECURITYSCHEME_FLOW._serialized_start=5093 - _SECURITYSCHEME_FLOW._serialized_end=5199 - _SECURITYREQUIREMENT._serialized_start=5202 - _SECURITYREQUIREMENT._serialized_end=5531 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_start=5342 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE._serialized_end=5383 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_start=5386 - _SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY._serialized_end=5531 - _SCOPES._serialized_start=5534 - _SCOPES._serialized_end=5663 - _SCOPES_SCOPEENTRY._serialized_start=5619 - _SCOPES_SCOPEENTRY._serialized_end=5663 +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protoc_gen_swagger.options.openapiv2_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'ZAgithub.amrom.workers.dev/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options' + _globals['_SWAGGER_RESPONSESENTRY']._loaded_options = None + _globals['_SWAGGER_RESPONSESENTRY']._serialized_options = b'8\001' + _globals['_SWAGGER_EXTENSIONSENTRY']._loaded_options = None + _globals['_SWAGGER_EXTENSIONSENTRY']._serialized_options = b'8\001' + _globals['_OPERATION_RESPONSESENTRY']._loaded_options = None + _globals['_OPERATION_RESPONSESENTRY']._serialized_options = b'8\001' + _globals['_OPERATION_EXTENSIONSENTRY']._loaded_options = None + _globals['_OPERATION_EXTENSIONSENTRY']._serialized_options = b'8\001' + _globals['_RESPONSE_HEADERSENTRY']._loaded_options = None + _globals['_RESPONSE_HEADERSENTRY']._serialized_options = b'8\001' + _globals['_RESPONSE_EXAMPLESENTRY']._loaded_options = None + _globals['_RESPONSE_EXAMPLESENTRY']._serialized_options = b'8\001' + _globals['_RESPONSE_EXTENSIONSENTRY']._loaded_options = None + _globals['_RESPONSE_EXTENSIONSENTRY']._serialized_options = b'8\001' + _globals['_INFO_EXTENSIONSENTRY']._loaded_options = None + _globals['_INFO_EXTENSIONSENTRY']._serialized_options = b'8\001' + _globals['_SCHEMA'].fields_by_name['example']._loaded_options = None + _globals['_SCHEMA'].fields_by_name['example']._serialized_options = b'\030\001' + _globals['_SECURITYDEFINITIONS_SECURITYENTRY']._loaded_options = None + _globals['_SECURITYDEFINITIONS_SECURITYENTRY']._serialized_options = b'8\001' + _globals['_SECURITYSCHEME_EXTENSIONSENTRY']._loaded_options = None + _globals['_SECURITYSCHEME_EXTENSIONSENTRY']._serialized_options = b'8\001' + _globals['_SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY']._loaded_options = None + _globals['_SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY']._serialized_options = b'8\001' + _globals['_SCOPES_SCOPEENTRY']._loaded_options = None + _globals['_SCOPES_SCOPEENTRY']._serialized_options = b'8\001' + _globals['_SWAGGER']._serialized_start=145 + _globals['_SWAGGER']._serialized_end=1073 + _globals['_SWAGGER_RESPONSESENTRY']._serialized_start=813 + _globals['_SWAGGER_RESPONSESENTRY']._serialized_end=912 + _globals['_SWAGGER_EXTENSIONSENTRY']._serialized_start=914 + _globals['_SWAGGER_EXTENSIONSENTRY']._serialized_end=987 + _globals['_SWAGGER_SWAGGERSCHEME']._serialized_start=989 + _globals['_SWAGGER_SWAGGERSCHEME']._serialized_end=1055 + _globals['_OPERATION']._serialized_start=1076 + _globals['_OPERATION']._serialized_end=1757 + _globals['_OPERATION_RESPONSESENTRY']._serialized_start=813 + _globals['_OPERATION_RESPONSESENTRY']._serialized_end=912 + _globals['_OPERATION_EXTENSIONSENTRY']._serialized_start=914 + _globals['_OPERATION_EXTENSIONSENTRY']._serialized_end=987 + _globals['_HEADER']._serialized_start=1760 + _globals['_HEADER']._serialized_end=1931 + _globals['_RESPONSE']._serialized_start=1934 + _globals['_RESPONSE']._serialized_end=2502 + _globals['_RESPONSE_HEADERSENTRY']._serialized_start=2283 + _globals['_RESPONSE_HEADERSENTRY']._serialized_end=2378 + _globals['_RESPONSE_EXAMPLESENTRY']._serialized_start=2380 + _globals['_RESPONSE_EXAMPLESENTRY']._serialized_end=2427 + _globals['_RESPONSE_EXTENSIONSENTRY']._serialized_start=914 + _globals['_RESPONSE_EXTENSIONSENTRY']._serialized_end=987 + _globals['_INFO']._serialized_start=2505 + _globals['_INFO']._serialized_end=2882 + _globals['_INFO_EXTENSIONSENTRY']._serialized_start=914 + _globals['_INFO_EXTENSIONSENTRY']._serialized_end=987 + _globals['_CONTACT']._serialized_start=2884 + _globals['_CONTACT']._serialized_end=2935 + _globals['_LICENSE']._serialized_start=2937 + _globals['_LICENSE']._serialized_end=2973 + _globals['_EXTERNALDOCUMENTATION']._serialized_start=2975 + _globals['_EXTERNALDOCUMENTATION']._serialized_end=3032 + _globals['_SCHEMA']._serialized_start=3035 + _globals['_SCHEMA']._serialized_end=3319 + _globals['_JSONSCHEMA']._serialized_start=3322 + _globals['_JSONSCHEMA']._serialized_end=4061 + _globals['_JSONSCHEMA_JSONSCHEMASIMPLETYPES']._serialized_start=3864 + _globals['_JSONSCHEMA_JSONSCHEMASIMPLETYPES']._serialized_end=3983 + _globals['_TAG']._serialized_start=4063 + _globals['_TAG']._serialized_end=4182 + _globals['_SECURITYDEFINITIONS']._serialized_start=4185 + _globals['_SECURITYDEFINITIONS']._serialized_end=4406 + _globals['_SECURITYDEFINITIONS_SECURITYENTRY']._serialized_start=4302 + _globals['_SECURITYDEFINITIONS_SECURITYENTRY']._serialized_end=4406 + _globals['_SECURITYSCHEME']._serialized_start=4409 + _globals['_SECURITYSCHEME']._serialized_end=5199 + _globals['_SECURITYSCHEME_EXTENSIONSENTRY']._serialized_start=914 + _globals['_SECURITYSCHEME_EXTENSIONSENTRY']._serialized_end=987 + _globals['_SECURITYSCHEME_TYPE']._serialized_start=4965 + _globals['_SECURITYSCHEME_TYPE']._serialized_end=5040 + _globals['_SECURITYSCHEME_IN']._serialized_start=5042 + _globals['_SECURITYSCHEME_IN']._serialized_end=5091 + _globals['_SECURITYSCHEME_FLOW']._serialized_start=5093 + _globals['_SECURITYSCHEME_FLOW']._serialized_end=5199 + _globals['_SECURITYREQUIREMENT']._serialized_start=5202 + _globals['_SECURITYREQUIREMENT']._serialized_end=5531 + _globals['_SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE']._serialized_start=5342 + _globals['_SECURITYREQUIREMENT_SECURITYREQUIREMENTVALUE']._serialized_end=5383 + _globals['_SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY']._serialized_start=5386 + _globals['_SECURITYREQUIREMENT_SECURITYREQUIREMENTENTRY']._serialized_end=5531 + _globals['_SCOPES']._serialized_start=5534 + _globals['_SCOPES']._serialized_end=5663 + _globals['_SCOPES_SCOPEENTRY']._serialized_start=5619 + _globals['_SCOPES_SCOPEENTRY']._serialized_end=5663 # @@protoc_insertion_point(module_scope) diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2.pyi b/src/protoc_gen_swagger/options/openapiv2_pb2.pyi new file mode 100644 index 00000000..b8bfbaf0 --- /dev/null +++ b/src/protoc_gen_swagger/options/openapiv2_pb2.pyi @@ -0,0 +1,1317 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import collections.abc +import google.protobuf.any_pb2 +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import google.protobuf.struct_pb2 +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing.final +class Swagger(google.protobuf.message.Message): + """`Swagger` is a representation of OpenAPI v2 specification's Swagger object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#swaggerObject + + Example: + + option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "Echo API"; + version: "1.0"; + description: "; + contact: { + name: "gRPC-Gateway project"; + url: "https://github.com/grpc-ecosystem/grpc-gateway"; + email: "none@example.com"; + }; + license: { + name: "BSD 3-Clause License"; + url: "https://github.com/grpc-ecosystem/grpc-gateway/blob/master/LICENSE.txt"; + }; + }; + schemes: HTTPS; + consumes: "application/json"; + produces: "application/json"; + }; + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class _SwaggerScheme: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _SwaggerSchemeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[Swagger._SwaggerScheme.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + UNKNOWN: Swagger._SwaggerScheme.ValueType # 0 + HTTP: Swagger._SwaggerScheme.ValueType # 1 + HTTPS: Swagger._SwaggerScheme.ValueType # 2 + WS: Swagger._SwaggerScheme.ValueType # 3 + WSS: Swagger._SwaggerScheme.ValueType # 4 + + class SwaggerScheme(_SwaggerScheme, metaclass=_SwaggerSchemeEnumTypeWrapper): ... + UNKNOWN: Swagger.SwaggerScheme.ValueType # 0 + HTTP: Swagger.SwaggerScheme.ValueType # 1 + HTTPS: Swagger.SwaggerScheme.ValueType # 2 + WS: Swagger.SwaggerScheme.ValueType # 3 + WSS: Swagger.SwaggerScheme.ValueType # 4 + + @typing.final + class ResponsesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___Response: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___Response | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing.final + class ExtensionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> google.protobuf.struct_pb2.Value: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: google.protobuf.struct_pb2.Value | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + SWAGGER_FIELD_NUMBER: builtins.int + INFO_FIELD_NUMBER: builtins.int + HOST_FIELD_NUMBER: builtins.int + BASE_PATH_FIELD_NUMBER: builtins.int + SCHEMES_FIELD_NUMBER: builtins.int + CONSUMES_FIELD_NUMBER: builtins.int + PRODUCES_FIELD_NUMBER: builtins.int + RESPONSES_FIELD_NUMBER: builtins.int + SECURITY_DEFINITIONS_FIELD_NUMBER: builtins.int + SECURITY_FIELD_NUMBER: builtins.int + EXTERNAL_DOCS_FIELD_NUMBER: builtins.int + EXTENSIONS_FIELD_NUMBER: builtins.int + swagger: builtins.str + """Specifies the Swagger Specification version being used. It can be + used by the Swagger UI and other clients to interpret the API listing. The + value MUST be "2.0". + """ + host: builtins.str + """The host (name or ip) serving the API. This MUST be the host only and does + not include the scheme nor sub-paths. It MAY include a port. If the host is + not included, the host serving the documentation is to be used (including + the port). The host does not support path templating. + """ + base_path: builtins.str + """The base path on which the API is served, which is relative to the host. If + it is not included, the API is served directly under the host. The value + MUST start with a leading slash (/). The basePath does not support path + templating. + Note that using `base_path` does not change the endpoint paths that are + generated in the resulting Swagger file. If you wish to use `base_path` + with relatively generated Swagger paths, the `base_path` prefix must be + manually removed from your `google.api.http` paths and your code changed to + serve the API from the `base_path`. + """ + @property + def info(self) -> global___Info: + """Provides metadata about the API. The metadata can be used by the + clients if needed. + """ + + @property + def schemes(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[global___Swagger.SwaggerScheme.ValueType]: + """The transfer protocol of the API. Values MUST be from the list: "http", + "https", "ws", "wss". If the schemes is not included, the default scheme to + be used is the one used to access the Swagger definition itself. + """ + + @property + def consumes(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """A list of MIME types the APIs can consume. This is global to all APIs but + can be overridden on specific API calls. Value MUST be as described under + Mime Types. + """ + + @property + def produces(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """A list of MIME types the APIs can produce. This is global to all APIs but + can be overridden on specific API calls. Value MUST be as described under + Mime Types. + """ + + @property + def responses(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___Response]: + """An object to hold responses that can be used across operations. This + property does not define global responses for all operations. + """ + + @property + def security_definitions(self) -> global___SecurityDefinitions: + """Security scheme definitions that can be used across the specification.""" + + @property + def security(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SecurityRequirement]: + """A declaration of which security schemes are applied for the API as a whole. + The list of values describes alternative security schemes that can be used + (that is, there is a logical OR between the security requirements). + Individual operations can override this definition. + """ + + @property + def external_docs(self) -> global___ExternalDocumentation: + """Additional external documentation.""" + + @property + def extensions(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, google.protobuf.struct_pb2.Value]: ... + def __init__( + self, + *, + swagger: builtins.str = ..., + info: global___Info | None = ..., + host: builtins.str = ..., + base_path: builtins.str = ..., + schemes: collections.abc.Iterable[global___Swagger.SwaggerScheme.ValueType] | None = ..., + consumes: collections.abc.Iterable[builtins.str] | None = ..., + produces: collections.abc.Iterable[builtins.str] | None = ..., + responses: collections.abc.Mapping[builtins.str, global___Response] | None = ..., + security_definitions: global___SecurityDefinitions | None = ..., + security: collections.abc.Iterable[global___SecurityRequirement] | None = ..., + external_docs: global___ExternalDocumentation | None = ..., + extensions: collections.abc.Mapping[builtins.str, google.protobuf.struct_pb2.Value] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["external_docs", b"external_docs", "info", b"info", "security_definitions", b"security_definitions"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["base_path", b"base_path", "consumes", b"consumes", "extensions", b"extensions", "external_docs", b"external_docs", "host", b"host", "info", b"info", "produces", b"produces", "responses", b"responses", "schemes", b"schemes", "security", b"security", "security_definitions", b"security_definitions", "swagger", b"swagger"]) -> None: ... + +global___Swagger = Swagger + +@typing.final +class Operation(google.protobuf.message.Message): + """`Operation` is a representation of OpenAPI v2 specification's Operation object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#operationObject + + Example: + + service EchoService { + rpc Echo(SimpleMessage) returns (SimpleMessage) { + option (google.api.http) = { + get: "/v1/example/echo/{id}" + }; + + option (grpc.gateway.protoc_gen_swagger.options.openapiv2_operation) = { + summary: "Get a message."; + operation_id: "getMessage"; + tags: "echo"; + responses: { + key: "200" + value: { + description: "OK"; + } + } + }; + } + } + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing.final + class ResponsesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___Response: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___Response | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing.final + class ExtensionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> google.protobuf.struct_pb2.Value: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: google.protobuf.struct_pb2.Value | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + TAGS_FIELD_NUMBER: builtins.int + SUMMARY_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int + EXTERNAL_DOCS_FIELD_NUMBER: builtins.int + OPERATION_ID_FIELD_NUMBER: builtins.int + CONSUMES_FIELD_NUMBER: builtins.int + PRODUCES_FIELD_NUMBER: builtins.int + RESPONSES_FIELD_NUMBER: builtins.int + SCHEMES_FIELD_NUMBER: builtins.int + DEPRECATED_FIELD_NUMBER: builtins.int + SECURITY_FIELD_NUMBER: builtins.int + EXTENSIONS_FIELD_NUMBER: builtins.int + summary: builtins.str + """A short summary of what the operation does. For maximum readability in the + swagger-ui, this field SHOULD be less than 120 characters. + """ + description: builtins.str + """A verbose explanation of the operation behavior. GFM syntax can be used for + rich text representation. + """ + operation_id: builtins.str + """Unique string used to identify the operation. The id MUST be unique among + all operations described in the API. Tools and libraries MAY use the + operationId to uniquely identify an operation, therefore, it is recommended + to follow common programming naming conventions. + """ + deprecated: builtins.bool + """Declares this operation to be deprecated. Usage of the declared operation + should be refrained. Default value is false. + """ + @property + def tags(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """A list of tags for API documentation control. Tags can be used for logical + grouping of operations by resources or any other qualifier. + """ + + @property + def external_docs(self) -> global___ExternalDocumentation: + """Additional external documentation for this operation.""" + + @property + def consumes(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """A list of MIME types the operation can consume. This overrides the consumes + definition at the Swagger Object. An empty value MAY be used to clear the + global definition. Value MUST be as described under Mime Types. + """ + + @property + def produces(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """A list of MIME types the operation can produce. This overrides the produces + definition at the Swagger Object. An empty value MAY be used to clear the + global definition. Value MUST be as described under Mime Types. + """ + + @property + def responses(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___Response]: + """The list of possible responses as they are returned from executing this + operation. + """ + + @property + def schemes(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """The transfer protocol for the operation. Values MUST be from the list: + "http", "https", "ws", "wss". The value overrides the Swagger Object + schemes definition. + """ + + @property + def security(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___SecurityRequirement]: + """A declaration of which security schemes are applied for this operation. The + list of values describes alternative security schemes that can be used + (that is, there is a logical OR between the security requirements). This + definition overrides any declared top-level security. To remove a top-level + security declaration, an empty array can be used. + """ + + @property + def extensions(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, google.protobuf.struct_pb2.Value]: ... + def __init__( + self, + *, + tags: collections.abc.Iterable[builtins.str] | None = ..., + summary: builtins.str = ..., + description: builtins.str = ..., + external_docs: global___ExternalDocumentation | None = ..., + operation_id: builtins.str = ..., + consumes: collections.abc.Iterable[builtins.str] | None = ..., + produces: collections.abc.Iterable[builtins.str] | None = ..., + responses: collections.abc.Mapping[builtins.str, global___Response] | None = ..., + schemes: collections.abc.Iterable[builtins.str] | None = ..., + deprecated: builtins.bool = ..., + security: collections.abc.Iterable[global___SecurityRequirement] | None = ..., + extensions: collections.abc.Mapping[builtins.str, google.protobuf.struct_pb2.Value] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["external_docs", b"external_docs"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["consumes", b"consumes", "deprecated", b"deprecated", "description", b"description", "extensions", b"extensions", "external_docs", b"external_docs", "operation_id", b"operation_id", "produces", b"produces", "responses", b"responses", "schemes", b"schemes", "security", b"security", "summary", b"summary", "tags", b"tags"]) -> None: ... + +global___Operation = Operation + +@typing.final +class Header(google.protobuf.message.Message): + """`Header` is a representation of OpenAPI v2 specification's Header object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#headerObject + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DESCRIPTION_FIELD_NUMBER: builtins.int + TYPE_FIELD_NUMBER: builtins.int + FORMAT_FIELD_NUMBER: builtins.int + DEFAULT_FIELD_NUMBER: builtins.int + PATTERN_FIELD_NUMBER: builtins.int + description: builtins.str + """`Description` is a short description of the header.""" + type: builtins.str + """The type of the object. The value MUST be one of "string", "number", "integer", or "boolean". The "array" type is not supported.""" + format: builtins.str + """`Format` The extending format for the previously mentioned type.""" + default: builtins.str + """`Default` Declares the value of the header that the server will use if none is provided. + See: https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-6.2. + Unlike JSON Schema this value MUST conform to the defined type for the header. + """ + pattern: builtins.str + """'Pattern' See https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-5.2.3.""" + def __init__( + self, + *, + description: builtins.str = ..., + type: builtins.str = ..., + format: builtins.str = ..., + default: builtins.str = ..., + pattern: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["default", b"default", "description", b"description", "format", b"format", "pattern", b"pattern", "type", b"type"]) -> None: ... + +global___Header = Header + +@typing.final +class Response(google.protobuf.message.Message): + """`Response` is a representation of OpenAPI v2 specification's Response object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#responseObject + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing.final + class HeadersEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___Header: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___Header | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing.final + class ExamplesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing.final + class ExtensionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> google.protobuf.struct_pb2.Value: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: google.protobuf.struct_pb2.Value | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + DESCRIPTION_FIELD_NUMBER: builtins.int + SCHEMA_FIELD_NUMBER: builtins.int + HEADERS_FIELD_NUMBER: builtins.int + EXAMPLES_FIELD_NUMBER: builtins.int + EXTENSIONS_FIELD_NUMBER: builtins.int + description: builtins.str + """`Description` is a short description of the response. + GFM syntax can be used for rich text representation. + """ + @property + def schema(self) -> global___Schema: + """`Schema` optionally defines the structure of the response. + If `Schema` is not provided, it means there is no content to the response. + """ + + @property + def headers(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___Header]: + """`Headers` A list of headers that are sent with the response. + `Header` name is expected to be a string in the canonical format of the MIME header key + See: https://golang.org/pkg/net/textproto/#CanonicalMIMEHeaderKey + """ + + @property + def examples(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: + """`Examples` gives per-mimetype response examples. + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#example-object + """ + + @property + def extensions(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, google.protobuf.struct_pb2.Value]: ... + def __init__( + self, + *, + description: builtins.str = ..., + schema: global___Schema | None = ..., + headers: collections.abc.Mapping[builtins.str, global___Header] | None = ..., + examples: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + extensions: collections.abc.Mapping[builtins.str, google.protobuf.struct_pb2.Value] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["schema", b"schema"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["description", b"description", "examples", b"examples", "extensions", b"extensions", "headers", b"headers", "schema", b"schema"]) -> None: ... + +global___Response = Response + +@typing.final +class Info(google.protobuf.message.Message): + """`Info` is a representation of OpenAPI v2 specification's Info object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#infoObject + + Example: + + option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + title: "Echo API"; + version: "1.0"; + description: "; + contact: { + name: "gRPC-Gateway project"; + url: "https://github.com/grpc-ecosystem/grpc-gateway"; + email: "none@example.com"; + }; + license: { + name: "BSD 3-Clause License"; + url: "https://github.com/grpc-ecosystem/grpc-gateway/blob/master/LICENSE.txt"; + }; + }; + ... + }; + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing.final + class ExtensionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> google.protobuf.struct_pb2.Value: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: google.protobuf.struct_pb2.Value | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + TITLE_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int + TERMS_OF_SERVICE_FIELD_NUMBER: builtins.int + CONTACT_FIELD_NUMBER: builtins.int + LICENSE_FIELD_NUMBER: builtins.int + VERSION_FIELD_NUMBER: builtins.int + EXTENSIONS_FIELD_NUMBER: builtins.int + title: builtins.str + """The title of the application.""" + description: builtins.str + """A short description of the application. GFM syntax can be used for rich + text representation. + """ + terms_of_service: builtins.str + """The Terms of Service for the API.""" + version: builtins.str + """Provides the version of the application API (not to be confused + with the specification version). + """ + @property + def contact(self) -> global___Contact: + """The contact information for the exposed API.""" + + @property + def license(self) -> global___License: + """The license information for the exposed API.""" + + @property + def extensions(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, google.protobuf.struct_pb2.Value]: ... + def __init__( + self, + *, + title: builtins.str = ..., + description: builtins.str = ..., + terms_of_service: builtins.str = ..., + contact: global___Contact | None = ..., + license: global___License | None = ..., + version: builtins.str = ..., + extensions: collections.abc.Mapping[builtins.str, google.protobuf.struct_pb2.Value] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["contact", b"contact", "license", b"license"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["contact", b"contact", "description", b"description", "extensions", b"extensions", "license", b"license", "terms_of_service", b"terms_of_service", "title", b"title", "version", b"version"]) -> None: ... + +global___Info = Info + +@typing.final +class Contact(google.protobuf.message.Message): + """`Contact` is a representation of OpenAPI v2 specification's Contact object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#contactObject + + Example: + + option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + ... + contact: { + name: "gRPC-Gateway project"; + url: "https://github.com/grpc-ecosystem/grpc-gateway"; + email: "none@example.com"; + }; + ... + }; + ... + }; + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAME_FIELD_NUMBER: builtins.int + URL_FIELD_NUMBER: builtins.int + EMAIL_FIELD_NUMBER: builtins.int + name: builtins.str + """The identifying name of the contact person/organization.""" + url: builtins.str + """The URL pointing to the contact information. MUST be in the format of a + URL. + """ + email: builtins.str + """The email address of the contact person/organization. MUST be in the format + of an email address. + """ + def __init__( + self, + *, + name: builtins.str = ..., + url: builtins.str = ..., + email: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["email", b"email", "name", b"name", "url", b"url"]) -> None: ... + +global___Contact = Contact + +@typing.final +class License(google.protobuf.message.Message): + """`License` is a representation of OpenAPI v2 specification's License object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#licenseObject + + Example: + + option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + info: { + ... + license: { + name: "BSD 3-Clause License"; + url: "https://github.com/grpc-ecosystem/grpc-gateway/blob/master/LICENSE.txt"; + }; + ... + }; + ... + }; + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAME_FIELD_NUMBER: builtins.int + URL_FIELD_NUMBER: builtins.int + name: builtins.str + """The license name used for the API.""" + url: builtins.str + """A URL to the license used for the API. MUST be in the format of a URL.""" + def __init__( + self, + *, + name: builtins.str = ..., + url: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["name", b"name", "url", b"url"]) -> None: ... + +global___License = License + +@typing.final +class ExternalDocumentation(google.protobuf.message.Message): + """`ExternalDocumentation` is a representation of OpenAPI v2 specification's + ExternalDocumentation object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#externalDocumentationObject + + Example: + + option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = { + ... + external_docs: { + description: "More about gRPC-Gateway"; + url: "https://github.com/grpc-ecosystem/grpc-gateway"; + } + ... + }; + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DESCRIPTION_FIELD_NUMBER: builtins.int + URL_FIELD_NUMBER: builtins.int + description: builtins.str + """A short description of the target documentation. GFM syntax can be used for + rich text representation. + """ + url: builtins.str + """The URL for the target documentation. Value MUST be in the format + of a URL. + """ + def __init__( + self, + *, + description: builtins.str = ..., + url: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["description", b"description", "url", b"url"]) -> None: ... + +global___ExternalDocumentation = ExternalDocumentation + +@typing.final +class Schema(google.protobuf.message.Message): + """`Schema` is a representation of OpenAPI v2 specification's Schema object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#schemaObject + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + JSON_SCHEMA_FIELD_NUMBER: builtins.int + DISCRIMINATOR_FIELD_NUMBER: builtins.int + READ_ONLY_FIELD_NUMBER: builtins.int + EXTERNAL_DOCS_FIELD_NUMBER: builtins.int + EXAMPLE_FIELD_NUMBER: builtins.int + EXAMPLE_STRING_FIELD_NUMBER: builtins.int + discriminator: builtins.str + """Adds support for polymorphism. The discriminator is the schema property + name that is used to differentiate between other schema that inherit this + schema. The property name used MUST be defined at this schema and it MUST + be in the required property list. When used, the value MUST be the name of + this schema or any schema that inherits it. + """ + read_only: builtins.bool + """Relevant only for Schema "properties" definitions. Declares the property as + "read only". This means that it MAY be sent as part of a response but MUST + NOT be sent as part of the request. Properties marked as readOnly being + true SHOULD NOT be in the required list of the defined schema. Default + value is false. + """ + example_string: builtins.str + """A free-form property to include a JSON example of this field. This is copied + verbatim to the output swagger.json. Quotes must be escaped. + """ + @property + def json_schema(self) -> global___JSONSchema: ... + @property + def external_docs(self) -> global___ExternalDocumentation: + """Additional external documentation for this schema.""" + + @property + def example(self) -> google.protobuf.any_pb2.Any: + """A free-form property to include an example of an instance for this schema. + Deprecated, please use example_string instead. + """ + + def __init__( + self, + *, + json_schema: global___JSONSchema | None = ..., + discriminator: builtins.str = ..., + read_only: builtins.bool = ..., + external_docs: global___ExternalDocumentation | None = ..., + example: google.protobuf.any_pb2.Any | None = ..., + example_string: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["example", b"example", "external_docs", b"external_docs", "json_schema", b"json_schema"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["discriminator", b"discriminator", "example", b"example", "example_string", b"example_string", "external_docs", b"external_docs", "json_schema", b"json_schema", "read_only", b"read_only"]) -> None: ... + +global___Schema = Schema + +@typing.final +class JSONSchema(google.protobuf.message.Message): + """`JSONSchema` represents properties from JSON Schema taken, and as used, in + the OpenAPI v2 spec. + + This includes changes made by OpenAPI v2. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#schemaObject + + See also: https://cswr.github.io/JsonSchema/spec/basic_types/, + https://github.com/json-schema-org/json-schema-spec/blob/master/schema.json + + Example: + + message SimpleMessage { + option (grpc.gateway.protoc_gen_swagger.options.openapiv2_schema) = { + json_schema: { + title: "SimpleMessage" + description: "A simple message." + required: ["id"] + } + }; + + // Id represents the message identifier. + string id = 1; [ + (grpc.gateway.protoc_gen_swagger.options.openapiv2_field) = { + {description: "The unique identifier of the simple message." + }]; + } + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class _JSONSchemaSimpleTypes: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _JSONSchemaSimpleTypesEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[JSONSchema._JSONSchemaSimpleTypes.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + UNKNOWN: JSONSchema._JSONSchemaSimpleTypes.ValueType # 0 + ARRAY: JSONSchema._JSONSchemaSimpleTypes.ValueType # 1 + BOOLEAN: JSONSchema._JSONSchemaSimpleTypes.ValueType # 2 + INTEGER: JSONSchema._JSONSchemaSimpleTypes.ValueType # 3 + NULL: JSONSchema._JSONSchemaSimpleTypes.ValueType # 4 + NUMBER: JSONSchema._JSONSchemaSimpleTypes.ValueType # 5 + OBJECT: JSONSchema._JSONSchemaSimpleTypes.ValueType # 6 + STRING: JSONSchema._JSONSchemaSimpleTypes.ValueType # 7 + + class JSONSchemaSimpleTypes(_JSONSchemaSimpleTypes, metaclass=_JSONSchemaSimpleTypesEnumTypeWrapper): ... + UNKNOWN: JSONSchema.JSONSchemaSimpleTypes.ValueType # 0 + ARRAY: JSONSchema.JSONSchemaSimpleTypes.ValueType # 1 + BOOLEAN: JSONSchema.JSONSchemaSimpleTypes.ValueType # 2 + INTEGER: JSONSchema.JSONSchemaSimpleTypes.ValueType # 3 + NULL: JSONSchema.JSONSchemaSimpleTypes.ValueType # 4 + NUMBER: JSONSchema.JSONSchemaSimpleTypes.ValueType # 5 + OBJECT: JSONSchema.JSONSchemaSimpleTypes.ValueType # 6 + STRING: JSONSchema.JSONSchemaSimpleTypes.ValueType # 7 + + REF_FIELD_NUMBER: builtins.int + TITLE_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int + DEFAULT_FIELD_NUMBER: builtins.int + READ_ONLY_FIELD_NUMBER: builtins.int + EXAMPLE_FIELD_NUMBER: builtins.int + MULTIPLE_OF_FIELD_NUMBER: builtins.int + MAXIMUM_FIELD_NUMBER: builtins.int + EXCLUSIVE_MAXIMUM_FIELD_NUMBER: builtins.int + MINIMUM_FIELD_NUMBER: builtins.int + EXCLUSIVE_MINIMUM_FIELD_NUMBER: builtins.int + MAX_LENGTH_FIELD_NUMBER: builtins.int + MIN_LENGTH_FIELD_NUMBER: builtins.int + PATTERN_FIELD_NUMBER: builtins.int + MAX_ITEMS_FIELD_NUMBER: builtins.int + MIN_ITEMS_FIELD_NUMBER: builtins.int + UNIQUE_ITEMS_FIELD_NUMBER: builtins.int + MAX_PROPERTIES_FIELD_NUMBER: builtins.int + MIN_PROPERTIES_FIELD_NUMBER: builtins.int + REQUIRED_FIELD_NUMBER: builtins.int + ARRAY_FIELD_NUMBER: builtins.int + TYPE_FIELD_NUMBER: builtins.int + FORMAT_FIELD_NUMBER: builtins.int + ENUM_FIELD_NUMBER: builtins.int + ref: builtins.str + """Ref is used to define an external reference to include in the message. + This could be a fully qualified proto message reference, and that type must + be imported into the protofile. If no message is identified, the Ref will + be used verbatim in the output. + For example: + `ref: ".google.protobuf.Timestamp"`. + """ + title: builtins.str + """The title of the schema.""" + description: builtins.str + """A short description of the schema.""" + default: builtins.str + read_only: builtins.bool + example: builtins.str + """A free-form property to include a JSON example of this field. This is copied + verbatim to the output swagger.json. Quotes must be escaped. + This property is the same for 2.0 and 3.0.0 https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/3.0.0.md#schemaObject https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#schemaObject + """ + multiple_of: builtins.float + maximum: builtins.float + """Maximum represents an inclusive upper limit for a numeric instance. The + value of MUST be a number, + """ + exclusive_maximum: builtins.bool + minimum: builtins.float + """minimum represents an inclusive lower limit for a numeric instance. The + value of MUST be a number, + """ + exclusive_minimum: builtins.bool + max_length: builtins.int + min_length: builtins.int + pattern: builtins.str + max_items: builtins.int + min_items: builtins.int + unique_items: builtins.bool + max_properties: builtins.int + min_properties: builtins.int + format: builtins.str + """`Format`""" + @property + def required(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + @property + def array(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Items in 'array' must be unique.""" + + @property + def type(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[global___JSONSchema.JSONSchemaSimpleTypes.ValueType]: ... + @property + def enum(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Items in `enum` must be unique https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-5.5.1""" + + def __init__( + self, + *, + ref: builtins.str = ..., + title: builtins.str = ..., + description: builtins.str = ..., + default: builtins.str = ..., + read_only: builtins.bool = ..., + example: builtins.str = ..., + multiple_of: builtins.float = ..., + maximum: builtins.float = ..., + exclusive_maximum: builtins.bool = ..., + minimum: builtins.float = ..., + exclusive_minimum: builtins.bool = ..., + max_length: builtins.int = ..., + min_length: builtins.int = ..., + pattern: builtins.str = ..., + max_items: builtins.int = ..., + min_items: builtins.int = ..., + unique_items: builtins.bool = ..., + max_properties: builtins.int = ..., + min_properties: builtins.int = ..., + required: collections.abc.Iterable[builtins.str] | None = ..., + array: collections.abc.Iterable[builtins.str] | None = ..., + type: collections.abc.Iterable[global___JSONSchema.JSONSchemaSimpleTypes.ValueType] | None = ..., + format: builtins.str = ..., + enum: collections.abc.Iterable[builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["array", b"array", "default", b"default", "description", b"description", "enum", b"enum", "example", b"example", "exclusive_maximum", b"exclusive_maximum", "exclusive_minimum", b"exclusive_minimum", "format", b"format", "max_items", b"max_items", "max_length", b"max_length", "max_properties", b"max_properties", "maximum", b"maximum", "min_items", b"min_items", "min_length", b"min_length", "min_properties", b"min_properties", "minimum", b"minimum", "multiple_of", b"multiple_of", "pattern", b"pattern", "read_only", b"read_only", "ref", b"ref", "required", b"required", "title", b"title", "type", b"type", "unique_items", b"unique_items"]) -> None: ... + +global___JSONSchema = JSONSchema + +@typing.final +class Tag(google.protobuf.message.Message): + """`Tag` is a representation of OpenAPI v2 specification's Tag object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#tagObject + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DESCRIPTION_FIELD_NUMBER: builtins.int + EXTERNAL_DOCS_FIELD_NUMBER: builtins.int + description: builtins.str + """A short description for the tag. GFM syntax can be used for rich text + representation. + """ + @property + def external_docs(self) -> global___ExternalDocumentation: + """Additional external documentation for this tag.""" + + def __init__( + self, + *, + description: builtins.str = ..., + external_docs: global___ExternalDocumentation | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["external_docs", b"external_docs"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["description", b"description", "external_docs", b"external_docs"]) -> None: ... + +global___Tag = Tag + +@typing.final +class SecurityDefinitions(google.protobuf.message.Message): + """`SecurityDefinitions` is a representation of OpenAPI v2 specification's + Security Definitions object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#securityDefinitionsObject + + A declaration of the security schemes available to be used in the + specification. This does not enforce the security schemes on the operations + and only serves to provide the relevant details for each scheme. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing.final + class SecurityEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___SecurityScheme: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___SecurityScheme | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + SECURITY_FIELD_NUMBER: builtins.int + @property + def security(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___SecurityScheme]: + """A single security scheme definition, mapping a "name" to the scheme it + defines. + """ + + def __init__( + self, + *, + security: collections.abc.Mapping[builtins.str, global___SecurityScheme] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["security", b"security"]) -> None: ... + +global___SecurityDefinitions = SecurityDefinitions + +@typing.final +class SecurityScheme(google.protobuf.message.Message): + """`SecurityScheme` is a representation of OpenAPI v2 specification's + Security Scheme object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#securitySchemeObject + + Allows the definition of a security scheme that can be used by the + operations. Supported schemes are basic authentication, an API key (either as + a header or as a query parameter) and OAuth2's common flows (implicit, + password, application and access code). + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class _Type: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[SecurityScheme._Type.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + TYPE_INVALID: SecurityScheme._Type.ValueType # 0 + TYPE_BASIC: SecurityScheme._Type.ValueType # 1 + TYPE_API_KEY: SecurityScheme._Type.ValueType # 2 + TYPE_OAUTH2: SecurityScheme._Type.ValueType # 3 + + class Type(_Type, metaclass=_TypeEnumTypeWrapper): + """The type of the security scheme. Valid values are "basic", + "apiKey" or "oauth2". + """ + + TYPE_INVALID: SecurityScheme.Type.ValueType # 0 + TYPE_BASIC: SecurityScheme.Type.ValueType # 1 + TYPE_API_KEY: SecurityScheme.Type.ValueType # 2 + TYPE_OAUTH2: SecurityScheme.Type.ValueType # 3 + + class _In: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _InEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[SecurityScheme._In.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + IN_INVALID: SecurityScheme._In.ValueType # 0 + IN_QUERY: SecurityScheme._In.ValueType # 1 + IN_HEADER: SecurityScheme._In.ValueType # 2 + + class In(_In, metaclass=_InEnumTypeWrapper): + """The location of the API key. Valid values are "query" or "header".""" + + IN_INVALID: SecurityScheme.In.ValueType # 0 + IN_QUERY: SecurityScheme.In.ValueType # 1 + IN_HEADER: SecurityScheme.In.ValueType # 2 + + class _Flow: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _FlowEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[SecurityScheme._Flow.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + FLOW_INVALID: SecurityScheme._Flow.ValueType # 0 + FLOW_IMPLICIT: SecurityScheme._Flow.ValueType # 1 + FLOW_PASSWORD: SecurityScheme._Flow.ValueType # 2 + FLOW_APPLICATION: SecurityScheme._Flow.ValueType # 3 + FLOW_ACCESS_CODE: SecurityScheme._Flow.ValueType # 4 + + class Flow(_Flow, metaclass=_FlowEnumTypeWrapper): + """The flow used by the OAuth2 security scheme. Valid values are + "implicit", "password", "application" or "accessCode". + """ + + FLOW_INVALID: SecurityScheme.Flow.ValueType # 0 + FLOW_IMPLICIT: SecurityScheme.Flow.ValueType # 1 + FLOW_PASSWORD: SecurityScheme.Flow.ValueType # 2 + FLOW_APPLICATION: SecurityScheme.Flow.ValueType # 3 + FLOW_ACCESS_CODE: SecurityScheme.Flow.ValueType # 4 + + @typing.final + class ExtensionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> google.protobuf.struct_pb2.Value: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: google.protobuf.struct_pb2.Value | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + TYPE_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + IN_FIELD_NUMBER: builtins.int + FLOW_FIELD_NUMBER: builtins.int + AUTHORIZATION_URL_FIELD_NUMBER: builtins.int + TOKEN_URL_FIELD_NUMBER: builtins.int + SCOPES_FIELD_NUMBER: builtins.int + EXTENSIONS_FIELD_NUMBER: builtins.int + type: global___SecurityScheme.Type.ValueType + """The type of the security scheme. Valid values are "basic", + "apiKey" or "oauth2". + """ + description: builtins.str + """A short description for security scheme.""" + name: builtins.str + """The name of the header or query parameter to be used. + Valid for apiKey. + """ + flow: global___SecurityScheme.Flow.ValueType + """The flow used by the OAuth2 security scheme. Valid values are + "implicit", "password", "application" or "accessCode". + Valid for oauth2. + """ + authorization_url: builtins.str + """The authorization URL to be used for this flow. This SHOULD be in + the form of a URL. + Valid for oauth2/implicit and oauth2/accessCode. + """ + token_url: builtins.str + """The token URL to be used for this flow. This SHOULD be in the + form of a URL. + Valid for oauth2/password, oauth2/application and oauth2/accessCode. + """ + @property + def scopes(self) -> global___Scopes: + """The available scopes for the OAuth2 security scheme. + Valid for oauth2. + """ + + @property + def extensions(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, google.protobuf.struct_pb2.Value]: ... + def __init__( + self, + *, + type: global___SecurityScheme.Type.ValueType = ..., + description: builtins.str = ..., + name: builtins.str = ..., + flow: global___SecurityScheme.Flow.ValueType = ..., + authorization_url: builtins.str = ..., + token_url: builtins.str = ..., + scopes: global___Scopes | None = ..., + extensions: collections.abc.Mapping[builtins.str, google.protobuf.struct_pb2.Value] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["scopes", b"scopes"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["authorization_url", b"authorization_url", "description", b"description", "extensions", b"extensions", "flow", b"flow", "in", b"in", "name", b"name", "scopes", b"scopes", "token_url", b"token_url", "type", b"type"]) -> None: ... + +global___SecurityScheme = SecurityScheme + +@typing.final +class SecurityRequirement(google.protobuf.message.Message): + """`SecurityRequirement` is a representation of OpenAPI v2 specification's + Security Requirement object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#securityRequirementObject + + Lists the required security schemes to execute this operation. The object can + have multiple security schemes declared in it which are all required (that + is, there is a logical AND between the schemes). + + The name used for each property MUST correspond to a security scheme + declared in the Security Definitions. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing.final + class SecurityRequirementValue(google.protobuf.message.Message): + """If the security scheme is of type "oauth2", then the value is a list of + scope names required for the execution. For other security scheme types, + the array MUST be empty. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SCOPE_FIELD_NUMBER: builtins.int + @property + def scope(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + def __init__( + self, + *, + scope: collections.abc.Iterable[builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["scope", b"scope"]) -> None: ... + + @typing.final + class SecurityRequirementEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + @property + def value(self) -> global___SecurityRequirement.SecurityRequirementValue: ... + def __init__( + self, + *, + key: builtins.str = ..., + value: global___SecurityRequirement.SecurityRequirementValue | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["value", b"value"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + SECURITY_REQUIREMENT_FIELD_NUMBER: builtins.int + @property + def security_requirement(self) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___SecurityRequirement.SecurityRequirementValue]: + """Each name must correspond to a security scheme which is declared in + the Security Definitions. If the security scheme is of type "oauth2", + then the value is a list of scope names required for the execution. + For other security scheme types, the array MUST be empty. + """ + + def __init__( + self, + *, + security_requirement: collections.abc.Mapping[builtins.str, global___SecurityRequirement.SecurityRequirementValue] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["security_requirement", b"security_requirement"]) -> None: ... + +global___SecurityRequirement = SecurityRequirement + +@typing.final +class Scopes(google.protobuf.message.Message): + """`Scopes` is a representation of OpenAPI v2 specification's Scopes object. + + See: https://github.com/OAI/OpenAPI-Specification/blob/3.0.0/versions/2.0.md#scopesObject + + Lists the available scopes for an OAuth2 security scheme. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing.final + class ScopeEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["key", b"key", "value", b"value"]) -> None: ... + + SCOPE_FIELD_NUMBER: builtins.int + @property + def scope(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: + """Maps between a name of a scope to a short description of it (as the value + of the property). + """ + + def __init__( + self, + *, + scope: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["scope", b"scope"]) -> None: ... + +global___Scopes = Scopes diff --git a/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py index 2daafffe..929d0253 100644 --- a/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py +++ b/src/protoc_gen_swagger/options/openapiv2_pb2_grpc.py @@ -1,4 +1,24 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc +import warnings + +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in protoc_gen_swagger/options/openapiv2_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 77eed9d1..2a69dd2a 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.32.0' +__version__ = '1.33.0' diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index b5c3c03c..addf36f2 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -1,8 +1,12 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" -import grpc import warnings +import grpc + +GRPC_GENERATED_VERSION = '1.73.1' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False GRPC_GENERATED_VERSION = '1.73.1' GRPC_VERSION = grpc.__version__ diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 983d773d..97bed902 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -34,7 +34,9 @@ from scanoss.cryptography import Cryptography, create_cryptography_config_from_args from scanoss.export.dependency_track import DependencyTrackExporter -from scanoss.inspection.dependency_track.project_violation import DependencyTrackProjectViolationPolicyCheck +from scanoss.inspection.dependency_track.project_violation import ( + DependencyTrackProjectViolationPolicyCheck, +) from scanoss.inspection.raw.component_summary import ComponentSummary from scanoss.inspection.raw.license_summary import LicenseSummary from scanoss.scanners.container_scanner import ( @@ -310,6 +312,16 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_vulns.set_defaults(func=comp_vulns) c_vulns.add_argument('--grpc', action='store_true', help='Enable gRPC support') + # Component Sub-command: component licenses + c_licenses = comp_sub.add_parser( + 'licenses', + aliases=['lics'], + description=f'Show License details: {__version__}', + help='Retrieve licenses for the given components', + ) + c_licenses.add_argument('--grpc', action='store_true', help='Enable gRPC support') + c_licenses.set_defaults(func=comp_licenses) + # Component Sub-command: component semgrep c_semgrep = comp_sub.add_parser( 'semgrep', @@ -411,7 +423,15 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_versions_in_range.set_defaults(func=crypto_versions_in_range) # Common purl Component sub-command options - for p in [c_vulns, c_semgrep, c_provenance, p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range]: + for p in [ + c_vulns, + c_semgrep, + c_provenance, + p_crypto_algorithms, + p_crypto_hints, + p_crypto_versions_in_range, + c_licenses, + ]: p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') @@ -425,6 +445,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range, + c_licenses, ]: p.add_argument( '--timeout', @@ -541,32 +562,32 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 # ========================================================================= # INSPECT SUBCOMMAND - Analysis and validation of scan results # ========================================================================= - + # Main inspect parser - provides tools for analyzing scan results p_inspect = subparsers.add_parser( - 'inspect', - aliases=['insp', 'ins'], + 'inspect', + aliases=['insp', 'ins'], description=f'Inspect and analyse scan results: {__version__}', - help='Inspect and analyse scan results' + help='Inspect and analyse scan results', ) # Inspect sub-commands parser p_inspect_sub = p_inspect.add_subparsers( - title='Inspect Commands', - dest='subparsercmd', - description='Available inspection sub-commands', - help='Choose an inspection type' + title='Inspect Commands', + dest='subparsercmd', + description='Available inspection sub-commands', + help='Choose an inspection type', ) # ------------------------------------------------------------------------- # RAW RESULTS INSPECTION - Analyse raw scan output # ------------------------------------------------------------------------- - + # Raw results parser - handles inspection of unprocessed scan results p_inspect_raw = p_inspect_sub.add_parser( 'raw', description='Inspect and analyse SCANOSS raw scan results', - help='Analyse raw scan results for various compliance issues' + help='Analyse raw scan results for various compliance issues', ) # Raw results sub-commands parser @@ -574,15 +595,15 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 title='Raw Results Inspection Commands', dest='subparser_subcmd', description='Tools for analyzing raw scan results', - help='Choose a raw results analysis type' + help='Choose a raw results analysis type', ) # Copyleft license inspection - identifies copyleft license violations p_inspect_raw_copyleft = p_inspect_raw_sub.add_parser( - 'copyleft', - aliases=['cp'], - description='Identify components with copyleft licenses that may require compliance action', - help='Find copyleft license violations' + 'copyleft', + aliases=['cp'], + description='Identify components with copyleft licenses that may require compliance action', + help='Find copyleft license violations', ) # License summary inspection - provides overview of all detected licenses @@ -590,7 +611,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 'license-summary', aliases=['lic-summary', 'licsum'], description='Generate comprehensive summary of all licenses found in scan results', - help='Generate license summary report' + help='Generate license summary report', ) # Component summary inspection - provides overview of all detected components @@ -598,7 +619,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 'component-summary', aliases=['comp-summary', 'compsum'], description='Generate comprehensive summary of all components found in scan results', - help='Generate component summary report' + help='Generate component summary report', ) # Undeclared components inspection - finds components not declared in SBOM @@ -606,7 +627,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 'undeclared', aliases=['un'], description='Identify components present in code but not declared in SBOM files', - help='Find undeclared components' + help='Find undeclared components', ) # SBOM format option for undeclared components inspection p_inspect_raw_undeclared.add_argument( @@ -614,19 +635,19 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 required=False, choices=['legacy', 'settings'], default='settings', - help='SBOM format type for comparison: legacy or settings (default)' + help='SBOM format type for comparison: legacy or settings (default)', ) # ------------------------------------------------------------------------- # BACKWARD COMPATIBILITY - Support old inspect command format # ------------------------------------------------------------------------- - + # Legacy copyleft inspection - backward compatibility for 'scanoss-py inspect copyleft' p_inspect_legacy_copyleft = p_inspect_sub.add_parser( - 'copyleft', - aliases=['cp'], - description='Identify components with copyleft licenses that may require compliance action', - help='Find copyleft license violations (legacy format)' + 'copyleft', + aliases=['cp'], + description='Identify components with copyleft licenses that may require compliance action', + help='Find copyleft license violations (legacy format)', ) # Legacy undeclared components inspection - backward compatibility for 'scanoss-py inspect undeclared' @@ -634,16 +655,16 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 'undeclared', aliases=['un'], description='Identify components present in code but not declared in SBOM files', - help='Find undeclared components (legacy format)' + help='Find undeclared components (legacy format)', ) - + # SBOM format option for legacy undeclared components inspection p_inspect_legacy_undeclared.add_argument( '--sbom-format', required=False, choices=['legacy', 'settings'], default='settings', - help='SBOM format type for comparison: legacy or settings (default)' + help='SBOM format type for comparison: legacy or settings (default)', ) # Legacy license summary inspection - backward compatibility for 'scanoss-py inspect license-summary' @@ -651,7 +672,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 'license-summary', aliases=['lic-summary', 'licsum'], description='Generate comprehensive summary of all licenses found in scan results', - help='Generate license summary report (legacy format)' + help='Generate license summary report (legacy format)', ) # Legacy component summary inspection - backward compatibility for 'scanoss-py inspect component-summary' @@ -659,83 +680,63 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 'component-summary', aliases=['comp-summary', 'compsum'], description='Generate comprehensive summary of all components found in scan results', - help='Generate component summary report (legacy format)' + help='Generate component summary report (legacy format)', ) # Applies the same configuration to both legacy and raw versions # License filtering options - common to (legacy) copyleft and license summary commands - for p in [p_inspect_raw_copyleft, p_inspect_raw_license_summary, - p_inspect_legacy_copyleft, p_inspect_legacy_license_summary]: - p.add_argument( - '--include', - help='Additional licenses to include in analysis (comma-separated list)' - ) - p.add_argument( - '--exclude', - help='Licenses to exclude from analysis (comma-separated list)' - ) - p.add_argument( - '--explicit', - help='Use only these specific licenses for analysis (comma-separated list)' - ) + for p in [ + p_inspect_raw_copyleft, + p_inspect_raw_license_summary, + p_inspect_legacy_copyleft, + p_inspect_legacy_license_summary, + ]: + p.add_argument('--include', help='Additional licenses to include in analysis (comma-separated list)') + p.add_argument('--exclude', help='Licenses to exclude from analysis (comma-separated list)') + p.add_argument('--explicit', help='Use only these specific licenses for analysis (comma-separated list)') # Common options for (legacy) copyleft and undeclared component inspection for p in [p_inspect_raw_copyleft, p_inspect_raw_undeclared, p_inspect_legacy_copyleft, p_inspect_legacy_undeclared]: + p.add_argument('-i', '--input', nargs='?', help='Path to scan results file to analyse') p.add_argument( - '-i', '--input', - nargs='?', - help='Path to scan results file to analyse' - ) - p.add_argument( - '-f', '--format', + '-f', + '--format', required=False, choices=['json', 'md', 'jira_md'], default='json', - help='Output format: json (default), md (Markdown), or jira_md (JIRA Markdown)' - ) - p.add_argument( - '-o', '--output', - type=str, - help='Save detailed results to specified file' - ) - p.add_argument( - '-s', '--status', - type=str, - help='Save summary status report to Markdown file' + help='Output format: json (default), md (Markdown), or jira_md (JIRA Markdown)', ) + p.add_argument('-o', '--output', type=str, help='Save detailed results to specified file') + p.add_argument('-s', '--status', type=str, help='Save summary status report to Markdown file') # Common options for (legacy) license and component summary commands - for p in [p_inspect_raw_license_summary, p_inspect_raw_component_summary, - p_inspect_legacy_license_summary, p_inspect_legacy_component_summary]: - p.add_argument( - '-i', '--input', - nargs='?', - help='Path to scan results file to analyse' - ) - p.add_argument( - '-o', '--output', - type=str, - help='Save summary report to specified file' - ) + for p in [ + p_inspect_raw_license_summary, + p_inspect_raw_component_summary, + p_inspect_legacy_license_summary, + p_inspect_legacy_component_summary, + ]: + p.add_argument('-i', '--input', nargs='?', help='Path to scan results file to analyse') + p.add_argument('-o', '--output', type=str, help='Save summary report to specified file') # ------------------------------------------------------------------------- # DEPENDENCY TRACK INSPECTION - Analyse Dependency Track project data # ------------------------------------------------------------------------- - + # Dependency Track parser - handles inspection of DT project status and violations p_dep_track_sub = p_inspect_sub.add_parser( 'dependency-track', aliases=['dt'], description='Inspect and analyse Dependency Track project status and policy violations', - help='Analyse Dependency Track projects' + help='Analyse Dependency Track projects', ) - + # Dependency Track sub-commands parser p_inspect_dep_track_sub = p_dep_track_sub.add_subparsers( title='Dependency Track Inspection Commands', dest='subparser_subcmd', description='Tools for analysing Dependency Track project data', - help='Choose a Dependency Track analysis type' + help='Choose a Dependency Track analysis type', ) # Project violations inspection - analyses policy violations in DT projects @@ -743,70 +744,52 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 'project-violations', aliases=['pv'], description='Analyse policy violations and compliance issues in Dependency Track projects', - help='Inspect project policy violations' + help='Inspect project policy violations', ) # Dependency Track connection and authentication options p_inspect_dt_project_violation.add_argument( - '--url', - required=True, - type=str, - help='Dependency Track server base URL (e.g., https://dtrack.example.com)' + '--url', required=True, type=str, help='Dependency Track server base URL (e.g., https://dtrack.example.com)' ) p_inspect_dt_project_violation.add_argument( - '--upload-token', '-ut', + '--upload-token', + '-ut', required=False, - type=str, - help='Project-specific upload token for accessing DT project data' + type=str, + help='Project-specific upload token for accessing DT project data', ) p_inspect_dt_project_violation.add_argument( - '--project-id', '-pid', - required=False, - type=str, - help='Dependency Track project UUID to inspect' + '--project-id', '-pid', required=False, type=str, help='Dependency Track project UUID to inspect' ) p_inspect_dt_project_violation.add_argument( - '--apikey', '-k', - required=True, - type=str, - help='Dependency Track API key for authentication' + '--apikey', '-k', required=True, type=str, help='Dependency Track API key for authentication' ) p_inspect_dt_project_violation.add_argument( - '--project-name', '-pn', - required=False, - type=str, - help='Dependency Track project name' + '--project-name', '-pn', required=False, type=str, help='Dependency Track project name' ) p_inspect_dt_project_violation.add_argument( - '--project-version', '-pv', - required=False, - type=str, - help='Dependency Track project version' + '--project-version', '-pv', required=False, type=str, help='Dependency Track project version' ) p_inspect_dt_project_violation.add_argument( - '--output', '-o', - required=False, - type=str, - help='Save inspection results to specified file' + '--output', '-o', required=False, type=str, help='Save inspection results to specified file' ) p_inspect_dt_project_violation.add_argument( - '--status', - required=False, - type=str, - help='Save summary status report to specified file' + '--status', required=False, type=str, help='Save summary status report to specified file' ) p_inspect_dt_project_violation.add_argument( - '--format', '-f', + '--format', + '-f', required=False, choices=['json', 'md', 'jira_md'], default='json', - help='Output format: json (default), md (Markdown) or jira_md (JIRA Markdown)' + help='Output format: json (default), md (Markdown) or jira_md (JIRA Markdown)', ) p_inspect_dt_project_violation.add_argument( - '--timeout', '-M', + '--timeout', + '-M', required=False, default=300, type=float, - help='Timeout (in seconds) for API communication (optional - default 300 sec)' + help='Timeout (in seconds) for API communication (optional - default 300 sec)', ) # TODO Move to the command call def location @@ -852,7 +835,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 e_dt.add_argument('-i', '--input', type=str, required=True, help='Input SBOM file (CycloneDX JSON format)') e_dt.add_argument('--url', type=str, required=True, help='Dependency Track base URL') e_dt.add_argument('--apikey', '-k', type=str, required=True, help='Dependency Track API key') - e_dt.add_argument('--output', '-o', type=str, help='File to save export token and uuid into') + e_dt.add_argument('--output', '-o', type=str, help='File to save export token and uuid into') e_dt.add_argument('--project-id', '-pid', type=str, help='Dependency Track project UUID') e_dt.add_argument('--project-name', '-pn', type=str, help='Dependency Track project name') e_dt.add_argument('--project-version', '-pv', type=str, help='Dependency Track project version') @@ -927,6 +910,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range, + c_licenses, ]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -1001,6 +985,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range, + c_licenses, ]: p.add_argument( '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' @@ -1039,6 +1024,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range, + c_licenses, ]: p.add_argument( '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' @@ -1104,6 +1090,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_algorithms, p_crypto_hints, p_crypto_versions_in_range, + c_licenses, e_dt, ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') @@ -1420,7 +1407,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, req_headers=process_req_headers(args.header), - use_grpc=args.grpc + use_grpc=args.grpc, ) if args.wfp: if not scanner.is_file_or_snippet_scan(): @@ -1554,13 +1541,14 @@ def convert(parser, args): # INSPECT COMMAND HANDLERS - Functions that execute inspection operations # ============================================================================= + def inspect_copyleft(parser, args): """ Handle copyleft license inspection command. - + Analyses scan results to identify components using copyleft licenses that may require compliance actions such as source code disclosure. - + Parameters ---------- parser : ArgumentParser @@ -1594,9 +1582,9 @@ def inspect_copyleft(parser, args): format_type=args.format, status=args.status, output=args.output, - include=args.include, # Additional licenses to check - exclude=args.exclude, # Licenses to ignore - explicit=args.explicit, # Explicit license list + include=args.include, # Additional licenses to check + exclude=args.exclude, # Licenses to ignore + explicit=args.explicit, # Explicit license list ) # Execute inspection and exit with appropriate status code @@ -1612,11 +1600,11 @@ def inspect_copyleft(parser, args): def inspect_undeclared(parser, args): """ Handle undeclared components inspection command. - + Analyses scan results to identify components that are present in the codebase but not declared in SBOM or manifest files, which may indicate security or compliance risks. - + Parameters ---------- parser : ArgumentParser @@ -1634,7 +1622,7 @@ def inspect_undeclared(parser, args): print_stderr('ERROR: Input file is required for undeclared component inspection') parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) sys.exit(1) - + # Initialise output file if specified if args.output: initialise_empty_file(args.output) @@ -1669,10 +1657,10 @@ def inspect_undeclared(parser, args): def inspect_license_summary(parser, args): """ Handle license summary inspection command. - + Generates comprehensive summary of all licenses detected in scan results, including license counts, risk levels, and compliance recommendations. - + Parameters ---------- parser : ArgumentParser @@ -1688,7 +1676,7 @@ def inspect_license_summary(parser, args): print_stderr('ERROR: Input file is required for license summary') parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) sys.exit(1) - + # Initialise output file if specified if args.output: initialise_empty_file(args.output) @@ -1700,9 +1688,9 @@ def inspect_license_summary(parser, args): quiet=args.quiet, filepath=args.input, output=args.output, - include=args.include, # Additional licenses to include - exclude=args.exclude, # Licenses to exclude from summary - explicit=args.explicit, # Explicit license list to summarize + include=args.include, # Additional licenses to include + exclude=args.exclude, # Licenses to exclude from summary + explicit=args.explicit, # Explicit license list to summarize ) try: # Execute summary generation @@ -1713,13 +1701,14 @@ def inspect_license_summary(parser, args): traceback.print_exc() sys.exit(1) + def inspect_component_summary(parser, args): """ Handle component summary inspection command. - + Generates a comprehensive summary of all components detected in scan results, including component counts, versions, match types, and security information. - + Parameters ---------- parser : ArgumentParser @@ -1734,10 +1723,10 @@ def inspect_component_summary(parser, args): print_stderr('ERROR: Input file is required for component summary') parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) sys.exit(1) - + # Initialise an output file if specified if args.output: - initialise_empty_file(args.output) # Create/clear output file + initialise_empty_file(args.output) # Create/clear output file # Create and configure component summary generator i_component_summary = ComponentSummary( @@ -1757,14 +1746,15 @@ def inspect_component_summary(parser, args): traceback.print_exc() sys.exit(1) + def inspect_dep_track_project_violations(parser, args): """ Handle Dependency Track project inspection command. - + Analyses Dependency Track projects for policy violations, security issues, and compliance status. Connects to DT API to retrieve project data and generate detailed violation reports. - + Parameters ---------- parser : ArgumentParser @@ -1794,14 +1784,14 @@ def inspect_dep_track_project_violations(parser, args): trace=args.trace, quiet=args.quiet, output=args.output, - status= args.status, + status=args.status, format_type=args.format, - url=args.url, # DT server URL - api_key=args.apikey, # Authentication key - project_id=args.project_id, # Target project UUID + url=args.url, # DT server URL + api_key=args.apikey, # Authentication key + project_id=args.project_id, # Target project UUID upload_token=args.upload_token, # Upload access token - project_name=args.project_name, # DT project name - project_version=args.project_version, # DT project version + project_name=args.project_name, # DT project name + project_version=args.project_version, # DT project version timeout=args.timeout, ) # Execute inspection and exit with appropriate status code @@ -1818,6 +1808,7 @@ def inspect_dep_track_project_violations(parser, args): # END INSPECT COMMAND HANDLERS # ============================================================================= + def export_dt(parser, args): """ Validates and exports a Software Bill of Materials (SBOM) to a Dependency-Track server. @@ -1845,8 +1836,9 @@ def export_dt(parser, args): trace=args.trace, quiet=args.quiet, ) - success = dt_exporter.upload_sbom_file(args.input, args.project_id, args.project_name, - args.project_version, args.output) + success = dt_exporter.upload_sbom_file( + args.input, args.project_id, args.project_name, args.project_version, args.output + ) if not success: sys.exit(1) except Exception as e: @@ -1855,6 +1847,7 @@ def export_dt(parser, args): traceback.print_exc() sys.exit(1) + def _dt_args_validator(parser, args): """ Validates command-line arguments related to project identification. @@ -1884,6 +1877,7 @@ def _dt_args_validator(parser, args): print_stderr('Please supply a project name (--project-name) and version (--project-version)') sys.exit(1) + def utils_certloc(*_): """ Run the "utils certloc" sub-command @@ -2307,6 +2301,42 @@ def comp_provenance(parser, args): sys.exit(1) +def comp_licenses(parser, args): + """ + Run the "component licenses" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('ERROR: Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'ERROR: Certificate file does not exist: {args.ca_cert}.') + sys.exit(1) + pac_file = get_pac_file(args.pac) + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.api2url, + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + grpc_proxy=args.grpc_proxy, + pac=pac_file, + timeout=args.timeout, + req_headers=process_req_headers(args.header), + use_grpc=args.grpc, + ) + if not comps.get_licenses(args.input, args.purl, args.output): + sys.exit(1) + + def results(parser, args): """ Run the "results" sub-command diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 6756fc82..25224fbe 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -29,6 +29,9 @@ from pypac.parser import PACFile +from scanoss.cyclonedx import CycloneDx +from scanoss.utils.file import validate_json_file + from .scanner import Scanner from .scanossbase import ScanossBase from .scanossgrpc import ScanossGrpc @@ -90,8 +93,9 @@ def __init__( # noqa: PLR0913, PLR0915 ignore_cert_errors=ignore_cert_errors, use_grpc=use_grpc, ) + self.cdx = CycloneDx(debug=self.debug) - def load_comps(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None)-> Optional[dict]: + def load_comps(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None) -> Optional[dict]: """ Load the specified components and return a dictionary @@ -101,8 +105,9 @@ def load_comps(self, json_file: Optional[str] = None, purls: Optional[List[str]] """ return self.load_purls(json_file, purls, 'components') - def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] = None, field:str = 'purls' - ) -> Optional[dict]: + def load_purls( + self, json_file: Optional[str] = None, purls: Optional[List[str]] = None, field: str = 'purls' + ) -> Optional[dict]: """ Load the specified purls and return a dictionary @@ -112,15 +117,15 @@ def load_purls(self, json_file: Optional[str] = None, purls: Optional[List[str]] :return: PURL Request dictionary or None """ if json_file: - if not os.path.isfile(json_file) or not os.access(json_file, os.R_OK): - self.print_stderr(f'ERROR: JSON file does not exist, is not a file, or is not readable: {json_file}') + result = validate_json_file(json_file) + if not result.is_valid: + self.print_stderr(f'ERROR: Problem parsing input JSON: {result.error}') return None - with open(json_file, 'r') as f: - try: - purl_request = json.loads(f.read()) - except Exception as e: - self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') - return None + + if self.cdx.is_cyclonedx_json(json.dumps(result.data)): + purl_request = self.cdx.get_purls_request_from_cdx(result.data, field) + else: + purl_request = result.data elif purls: if not all(isinstance(purl, str) for purl in purls): self.print_stderr('ERROR: PURLs must be a list of strings.') @@ -384,3 +389,35 @@ def get_provenance_details( self.print_msg(f'Results written to: {output_file}') self._close_file(output_file, file) return success + + def get_licenses(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: + """ + Retrieve the license details for the supplied PURLs + + Args: + json_file (str, optional): Input JSON file. Defaults to None. + purls (None, optional): PURLs to retrieve license details for. Defaults to None. + output_file (str, optional): Output file. Defaults to None. + + Returns: + bool: True on success, False otherwise + """ + success = False + + purls_request = self.load_purls(json_file, purls) + if not purls_request: + return False + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + + # We'll use the new ComponentBatchRequest instead of deprecated PurlRequest for the license api + component_batch_request = {'components': purls_request.get('purls')} + response = self.grpc_api.get_licenses(component_batch_request) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index 9f367ba3..7957609f 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from typing import Dict, List, Optional +from scanoss.cyclonedx import CycloneDx from scanoss.scanossbase import ScanossBase from scanoss.scanossgrpc import ScanossGrpc from scanoss.utils.abstract_presenter import AbstractPresenter @@ -26,6 +27,38 @@ class CryptographyConfig: quiet: bool = False with_range: bool = False + def _process_input_file(self) -> dict: + """ + Process and validate the input file, returning the validated purl_request. + + Returns: + dict: The validated purl_request dictionary + + Raises: + ScanossCryptographyError: If the input file is invalid + """ + result = validate_json_file(self.input_file) + if not result.is_valid: + raise ScanossCryptographyError( + f'There was a problem with the purl input file. {result.error}' + ) + + cdx = CycloneDx(debug=self.debug) + if cdx.is_cyclonedx_json(json.dumps(result.data)): + purl_request = cdx.get_purls_request_from_cdx(result.data) + else: + purl_request = result.data + + if ( + not isinstance(purl_request, dict) + or 'purls' not in purl_request + or not isinstance(purl_request['purls'], list) + or not all(isinstance(p, dict) and 'purl' in p for p in purl_request['purls']) + ): + raise ScanossCryptographyError('The supplied input file is not in the correct PurlRequest format.') + + return purl_request + def __post_init__(self): """ Validate that the configuration is valid. @@ -39,19 +72,8 @@ def __post_init__(self): f'Invalid PURL format: "{purl}".' f'It must include a version (e.g., pkg:type/name@version)' ) if self.input_file: - input_file_validation = validate_json_file(self.input_file) - if not input_file_validation.is_valid: - raise ScanossCryptographyError( - f'There was a problem with the purl input file. {input_file_validation.error}' - ) - if ( - not isinstance(input_file_validation.data, dict) - or 'purls' not in input_file_validation.data - or not isinstance(input_file_validation.data['purls'], list) - or not all(isinstance(p, dict) and 'purl' in p for p in input_file_validation.data['purls']) - ): - raise ScanossCryptographyError('The supplied input file is not in the correct PurlRequest format.') - purls = input_file_validation.data['purls'] + purl_request = self._process_input_file() + purls = purl_request['purls'] purls_with_requirement = [] if self.with_range and any('requirement' not in p for p in purls): raise ScanossCryptographyError( @@ -182,16 +204,10 @@ def get_versions_in_range(self) -> Optional[Dict]: return self.results - def _build_purls_request( - self, - ) -> Optional[dict]: + def _build_purls_request(self) -> Optional[dict]: """ Load the specified purls from a JSON file or a list of PURLs and return a dictionary - Args: - json_file (Optional[str], optional): The JSON file containing the PURLs. Defaults to None. - purls (Optional[List[str]], optional): The list of PURLs. Defaults to None. - Returns: Optional[dict]: The dictionary containing the PURLs """ diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index f3685a97..555ba4ad 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -394,6 +394,7 @@ def _sev_lookup(value: str): def is_cyclonedx_json(self, json_string: str) -> bool: """ Validate if the given JSON string is a valid CycloneDX JSON string + Args: json_string (str): JSON string to validate Returns: @@ -410,6 +411,27 @@ def is_cyclonedx_json(self, json_string: str) -> bool: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return False + def get_purls_request_from_cdx(self, cdx_dict: dict, field: str = 'purls') -> dict: + """ + Get the list of PURL requests (purl + requirement) from the given CDX dictionary + + Args: + cdx_dict (dict): CDX dictionary to parse + field (str): Field to extract from the CDX dictionary + Returns: + list[dict]: List of PURL requests (purl + requirement) + """ + components = cdx_dict.get('components', []) + parsed_purls = [] + for component in components: + version = component.get('version') + if version: + parsed_purls.append({'purl': component.get('purl'), 'requirement': version}) + else: + parsed_purls.append({'purl': component.get('purl')}) + purl_request = {field: parsed_purls} + return purl_request + # # End of CycloneDX Class diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index fc3d3be3..deed00b6 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -44,6 +44,8 @@ from pypac.resolver import ProxyResolver from urllib3.exceptions import InsecureRequestWarning +from scanoss.api.licenses.v2.scanoss_licenses_pb2 import ComponentsLicenseResponse +from scanoss.api.licenses.v2.scanoss_licenses_pb2_grpc import LicenseStub from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub from scanoss.constants import DEFAULT_TIMEOUT @@ -71,7 +73,9 @@ from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub -from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import ComponentsVulnerabilityResponse +from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import ( + ComponentsVulnerabilityResponse, +) from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub from .scanossbase import ScanossBase @@ -81,18 +85,20 @@ SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else '' DEFAULT_URI_PREFIX = '/v2' -MAX_CONCURRENT_REQUESTS = 5 # Maximum number of concurrent requests to make +MAX_CONCURRENT_REQUESTS = 5 # Maximum number of concurrent requests to make class ScanossGrpcError(Exception): """ Custom exception for SCANOSS gRPC errors """ + pass class ScanossGrpcStatusCode(IntEnum): """Status codes for SCANOSS gRPC responses""" + SUCCESS = 1 SUCCESS_WITH_WARNINGS = 2 FAILED_WITH_WARNINGS = 3 @@ -208,6 +214,7 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 self.vuln_stub = VulnerabilitiesStub(grpc.insecure_channel(self.url)) self.provenance_stub = GeoProvenanceStub(grpc.insecure_channel(self.url)) self.scanning_stub = ScanningStub(grpc.insecure_channel(self.url)) + self.license_stub = LicenseStub(grpc.insecure_channel(self.url)) else: if ca_cert is not None: credentials = grpc.ssl_channel_credentials(cert_data) # secure with specified certificate @@ -220,6 +227,7 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 self.vuln_stub = VulnerabilitiesStub(grpc.secure_channel(self.url, credentials)) self.provenance_stub = GeoProvenanceStub(grpc.secure_channel(self.url, credentials)) self.scanning_stub = ScanningStub(grpc.secure_channel(self.url, credentials)) + self.license_stub = LicenseStub(grpc.secure_channel(self.url, credentials)) @classmethod def _load_cert(cls, cert_file: str) -> bytes: @@ -700,7 +708,38 @@ def get_versions_in_range_for_purl(self, request: Dict) -> Optional[Dict]: 'Sending data for cryptographic versions in range decoration (rqId: {rqId})...', ) - def load_generic_headers(self, url): + def get_licenses(self, request: Dict) -> Optional[Dict]: + """ + Client function to call the rpc for Licenses GetComponentsLicenses + It will either use REST (default) or gRPC depending on the use_grpc flag + + Args: + request (Dict): ComponentsRequest + Returns: + Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull + """ + if self.use_grpc: + return self._get_licenses_grpc(request) + else: + return self._get_licenses_rest(request) + + def _get_licenses_grpc(self, request: Dict) -> Optional[Dict]: + """ + Client function to call the rpc for GetComponentsLicenses + + Args: + request (Dict): ComponentsRequest + Returns: + Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull + """ + return self._call_rpc( + self.license_stub.GetComponentsLicenses, + request, + ComponentsRequest, + 'Sending data for license decoration (rqId: {rqId})...', + ) + + def load_generic_headers(self, url: str = None): """ Adds custom headers from req_headers to metadata. @@ -764,6 +803,33 @@ def rest_post(self, uri: str, request_id: str, data: dict) -> dict: raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e return None + def _get_licenses_rest(self, purls: Dict) -> Optional[Dict]: + """ + Get the licenses for the given purls using REST API + + Args: + purls (Dict): Purl Request dictionary + Returns: + Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull + """ + if not purls: + self.print_stderr('ERROR: No message supplied to send to REST decoration service.') + return None + request_id = str(uuid.uuid4()) + self.print_debug(f'Sending data for Licenses via REST (request id: {request_id})...') + response = self.rest_post(f'{self.orig_url}{DEFAULT_URI_PREFIX}/licenses/components', request_id, purls) + self.print_trace(f'Received response for Licenses via REST (request id: {request_id}): {response}') + if response: + # Parse the JSON/Dict into the purl response + resp_obj = ParseDict(response, ComponentsLicenseResponse(), True) + if resp_obj: + self.print_debug(f'License Response: {resp_obj}') + if not self._check_status_response(resp_obj.status, request_id): + return None + del response['status'] + return response + return None + def _get_vulnerabilities_rest(self, purls: dict): """ Get the vulnerabilities for the given purls using REST API @@ -799,17 +865,20 @@ def _process_dep_file_rest(self, file, depth: int = 1) -> dict: request_id = str(uuid.uuid4()) self.print_debug(f'Sending data for Dependencies via REST (request id: {request_id})...') file_request = {'files': [file], 'depth': depth} - response = self.rest_post(f'{self.orig_url}{DEFAULT_URI_PREFIX}/dependencies/dependencies', - request_id, file_request - ) + response = self.rest_post( + f'{self.orig_url}{DEFAULT_URI_PREFIX}/dependencies/dependencies', request_id, file_request + ) self.print_trace(f'Received response for Dependencies via REST (request id: {request_id}): {response}') if response: return response return None + + # # End of ScanossGrpc Class # + @dataclass class GrpcConfig: url: str = DEFAULT_URL @@ -840,6 +909,7 @@ def create_grpc_config_from_args(args) -> GrpcConfig: grpc_proxy=getattr(args, 'grpc_proxy', None), ) + # # End of GrpcConfig class -# \ No newline at end of file +# diff --git a/tests/data/requirements.txt b/tests/data/requirements.txt index 6237a069..93683ed9 100644 --- a/tests/data/requirements.txt +++ b/tests/data/requirements.txt @@ -2,5 +2,5 @@ requests crc32c>=2.2 binaryornot progress -grpcio>1.42.0 -protobuf>3.19.1 +grpcio>=1.73.1 +protobuf>=6.3.1 From 26973c29d6c07c46363f2c3fd8b7a458e2ca9f46 Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Mon, 6 Oct 2025 10:03:55 +0200 Subject: [PATCH 391/489] [SP-3346] feat: allow REST on decoration services (#153) * [SP-3346] feat: add REST support for decoration services * [SP-3346] chore: pr comments * [SP-3346] fix: lint error * [SP-3346] chore: update changelog, bump version * [SP-3346] fix: license service api call * [SP-3346] fix: lint error * [SP-3346] fix: lint errors * [SP-3346] fix: retry errors * [SP-3346] chore: update changelog release date --- CHANGELOG.md | 5 + src/scanoss/__init__.py | 2 +- .../api/common/v2/scanoss_common_pb2.py | 14 +- .../components/v2/scanoss_components_pb2.py | 78 +- .../v2/scanoss_components_pb2_grpc.py | 12 +- .../v2/scanoss_cryptography_pb2.py | 136 +++- .../v2/scanoss_cryptography_pb2_grpc.py | 554 ++++++++++++- .../v2/scanoss_dependencies_pb2.py | 50 +- .../v2/scanoss_dependencies_pb2_grpc.py | 1 + .../v2/scanoss_geoprovenance_pb2.py | 70 +- .../v2/scanoss_geoprovenance_pb2_grpc.py | 190 ++++- .../api/licenses/v2/scanoss_licenses_pb2.py | 54 +- .../api/scanning/v2/scanoss_scanning_pb2.py | 36 +- .../api/semgrep/v2/scanoss_semgrep_pb2.py | 42 +- .../semgrep/v2/scanoss_semgrep_pb2_grpc.py | 110 ++- .../v2/scanoss_vulnerabilities_pb2.py | 42 +- src/scanoss/cli.py | 23 +- src/scanoss/components.py | 46 +- src/scanoss/cryptography.py | 58 +- src/scanoss/scanossgrpc.py | 751 ++++++++++-------- 20 files changed, 1646 insertions(+), 628 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 663c4eb3..9bb3fad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.34.0] - 2025-10-06 +### Added +- Add REST API support for decoration commands + ## [1.33.0] - 2025-09-19 ### Added - Add `licenses` sub-command to `component` command @@ -668,3 +672,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.31.5]: https://github.com/scanoss/scanoss.py/compare/v1.31.4...v1.31.5 [1.32.0]: https://github.com/scanoss/scanoss.py/compare/v1.31.5...v1.32.0 [1.33.0]: https://github.com/scanoss/scanoss.py/compare/v1.32.0...v1.33.0 +[1.34.0]: https://github.com/scanoss/scanoss.py/compare/v1.33.0...v1.34.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 2a69dd2a..3339f1a3 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.33.0' +__version__ = '1.34.0' diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2.py b/src/scanoss/api/common/v2/scanoss_common_pb2.py index a82eb997..3daa20b9 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2.py @@ -26,7 +26,7 @@ from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\x1a.protoc-gen-openapiv2/options/annotations.proto\x1a\x1fgoogle/api/field_behavior.proto\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"k\n\x10\x43omponentRequest\x12\x11\n\x04purl\x18\x01 \x01(\tB\x03\xe0\x41\x02\x12\x13\n\x0brequirement\x18\x02 \x01(\t:/\x92\x41,2*{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"}\"\xc8\x01\n\x11\x43omponentsRequest\x12@\n\ncomponents\x18\x01 \x03(\x0b\x32\'.scanoss.api.common.v2.ComponentRequestB\x03\xe0\x41\x02:q\x92\x41n2l{\"components\":[{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"},{\"purl\":\"pkg:github/scanoss/scanoss.py@v1.30.0\"}]}\"r\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\")\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*scanoss/api/common/v2/scanoss-common.proto\x12\x15scanoss.api.common.v2\x1a.protoc-gen-openapiv2/options/annotations.proto\x1a\x1fgoogle/api/field_behavior.proto\"T\n\x0eStatusResponse\x12\x31\n\x06status\x18\x01 \x01(\x0e\x32!.scanoss.api.common.v2.StatusCode\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1e\n\x0b\x45\x63hoRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0c\x45\x63hoResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"k\n\x10\x43omponentRequest\x12\x11\n\x04purl\x18\x01 \x01(\tB\x03\xe0\x41\x02\x12\x13\n\x0brequirement\x18\x02 \x01(\t:/\x92\x41,2*{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"}\"\xc8\x01\n\x11\x43omponentsRequest\x12@\n\ncomponents\x18\x01 \x03(\x0b\x32\'.scanoss.api.common.v2.ComponentRequestB\x03\xe0\x41\x02:q\x92\x41n2l{\"components\":[{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"},{\"purl\":\"pkg:github/scanoss/scanoss.py@v1.30.0\"}]}\"v\n\x0bPurlRequest\x12\x37\n\x05purls\x18\x01 \x03(\x0b\x32(.scanoss.api.common.v2.PurlRequest.Purls\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t:\x02\x18\x01\")\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t*`\n\nStatusCode\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0b\n\x07SUCCESS\x10\x01\x12\x1b\n\x17SUCCEEDED_WITH_WARNINGS\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06\x46\x41ILED\x10\x04\x42/Z-github.amrom.workers.dev/scanoss/papi/api/commonv2;commonv2b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -42,8 +42,10 @@ _globals['_COMPONENTSREQUEST'].fields_by_name['components']._serialized_options = b'\340A\002' _globals['_COMPONENTSREQUEST']._loaded_options = None _globals['_COMPONENTSREQUEST']._serialized_options = b'\222An2l{\"components\":[{\"purl\":\"pkg:github/scanoss/engine@1.0.0\"},{\"purl\":\"pkg:github/scanoss/scanoss.py@v1.30.0\"}]}' - _globals['_STATUSCODE']._serialized_start=772 - _globals['_STATUSCODE']._serialized_end=868 + _globals['_PURLREQUEST']._loaded_options = None + _globals['_PURLREQUEST']._serialized_options = b'\030\001' + _globals['_STATUSCODE']._serialized_start=776 + _globals['_STATUSCODE']._serialized_end=872 _globals['_STATUSRESPONSE']._serialized_start=150 _globals['_STATUSRESPONSE']._serialized_end=234 _globals['_ECHOREQUEST']._serialized_start=236 @@ -55,9 +57,9 @@ _globals['_COMPONENTSREQUEST']._serialized_start=411 _globals['_COMPONENTSREQUEST']._serialized_end=611 _globals['_PURLREQUEST']._serialized_start=613 - _globals['_PURLREQUEST']._serialized_end=727 + _globals['_PURLREQUEST']._serialized_end=731 _globals['_PURLREQUEST_PURLS']._serialized_start=685 _globals['_PURLREQUEST_PURLS']._serialized_end=727 - _globals['_PURL']._serialized_start=729 - _globals['_PURL']._serialized_end=770 + _globals['_PURL']._serialized_start=733 + _globals['_PURL']._serialized_end=774 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2.py b/src/scanoss/api/components/v2/scanoss_components_pb2.py index 3820ed92..fb2a523a 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2.py @@ -27,46 +27,60 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"v\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05\"\xca\x01\n\rCompStatistic\x12\x1a\n\x12total_source_files\x18\x01 \x01(\x05\x12\x13\n\x0btotal_lines\x18\x02 \x01(\x05\x12\x19\n\x11total_blank_lines\x18\x03 \x01(\x05\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05\"\xfb\x01\n\x15\x43ompStatisticResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompStatisticResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x64\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic\"\xd3\x01\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x39\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\"1\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05\"\xe4\x03\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aO\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1ar\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x12\x0c\n\x04\x64\x61te\x18\x05 \x01(\t\x1a\x83\x01\n\tComponent\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version2\xcb\x04\n\nComponents\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\x12\x17/api/v2/components/echo\x12\x92\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"!\x82\xd3\xe4\x93\x02\x1b\x12\x19/api/v2/components/search\x12\x9a\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"#\x82\xd3\xe4\x93\x02\x1d\x12\x1b/api/v2/components/versions\x12\x98\x01\n\x16GetComponentStatistics\x12\".scanoss.api.common.v2.PurlRequest\x1a\x30.scanoss.api.components.v2.CompStatisticResponse\"(\x82\xd3\xe4\x93\x02\"\"\x1d/api/v2/components/statistics:\x01*B\x9a\x03Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xdf\x02\x12\x9d\x01\n\x1aSCANOSS Components Service\x12(Provides component intelligence services\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x02\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07Z8\n6\n\x07\x61pi_key\x12+\x08\x02\x12\x1a\x41PI key for authentication\x1a\tx-api-key \x02\x62\r\n\x0b\n\x07\x61pi_key\x12\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2scanoss/api/components/v2/scanoss-components.proto\x12\x19scanoss.api.components.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xae\x01\n\x11\x43ompSearchRequest\x12\x0e\n\x06search\x18\x01 \x01(\t\x12\x0e\n\x06vendor\x18\x02 \x01(\t\x12\x11\n\tcomponent\x18\x03 \x01(\t\x12\x0f\n\x07package\x18\x04 \x01(\t\x12\r\n\x05limit\x18\x06 \x01(\x05\x12\x0e\n\x06offset\x18\x07 \x01(\x05:6\x92\x41\x33\n1J/{\"search\": \"scanoss\", \"limit\": 10, \"offset\": 0}\"\xfe\x01\n\rCompStatistic\x12.\n\x12total_source_files\x18\x01 \x01(\x05R\x12total_source_files\x12 \n\x0btotal_lines\x18\x02 \x01(\x05R\x0btotal_lines\x12,\n\x11total_blank_lines\x18\x03 \x01(\x05R\x11total_blank_lines\x12\x44\n\tlanguages\x18\x04 \x03(\x0b\x32\x31.scanoss.api.components.v2.CompStatistic.Language\x1a\'\n\x08Language\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x01(\x05\"\xb5\x05\n\x1b\x43omponentsStatisticResponse\x12~\n\x14\x63omponent_statistics\x18\x01 \x03(\x0b\x32J.scanoss.api.components.v2.ComponentsStatisticResponse.ComponentStatisticsR\x14\x63omponent_statistics\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1ar\n\x13\x43omponentStatistics\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12<\n\nstatistics\x18\x03 \x01(\x0b\x32(.scanoss.api.components.v2.CompStatistic:\xea\x02\x92\x41\xe6\x02\n\xe3\x02J\xe0\x02{\"component_statistics\": [{\"purl\": \"pkg:github/scanoss/engine@5.0.0\", \"version\": \"5.0.0\", \"statistics\": {\"total_source_files\": 156, \"total_lines\": 25430, \"total_blank_lines\": 3420, \"languages\": [{\"name\": \"C\", \"files\": 89}, {\"name\": \"C Header\", \"files\": 45}]}}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Component statistics successfully retrieved\"}}\"\xdb\x04\n\x12\x43ompSearchResponse\x12K\n\ncomponents\x18\x01 \x03(\x0b\x32\x37.scanoss.api.components.v2.CompSearchResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aK\n\tComponent\x12\x15\n\tcomponent\x18\x01 \x01(\tB\x02\x18\x01\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t:\xf3\x02\x92\x41\xef\x02\n\xec\x02J\xe9\x02{\"components\": [{\"name\": \"scanoss-py\", \"purl\": \"pkg:github/scanoss/scanoss.py\", \"url\": \"https://github.com/scanoss/scanoss.py\", \"component\": \"scanoss-py\"}, {\"name\": \"engine\", \"purl\": \"pkg:github/scanoss/engine\", \"url\": \"https://github.com/scanoss/engine\", \"component\": \"engine\"}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Components successfully retrieved\"}}\"l\n\x12\x43ompVersionRequest\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\r\n\x05limit\x18\x02 \x01(\x05:9\x92\x41\x36\n4J2{\"purl\": \"pkg:github/scanoss/engine\", \"limit\": 20}\"\xe0\x07\n\x13\x43ompVersionResponse\x12K\n\tcomponent\x18\x01 \x01(\x0b\x32\x38.scanoss.api.components.v2.CompVersionResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aj\n\x07License\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x18\n\x07spdx_id\x18\x02 \x01(\tR\x07spdx_id\x12*\n\x10is_spdx_approved\x18\x03 \x01(\x08R\x10is_spdx_approved\x12\x0b\n\x03url\x18\x04 \x01(\t\x1ar\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12H\n\x08licenses\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.License\x12\x0c\n\x04\x64\x61te\x18\x05 \x01(\t\x1a\x95\x01\n\tComponent\x12\x15\n\tcomponent\x18\x01 \x01(\tB\x02\x18\x01\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0b\n\x03url\x18\x03 \x01(\t\x12H\n\x08versions\x18\x04 \x03(\x0b\x32\x36.scanoss.api.components.v2.CompVersionResponse.Version\x12\x0c\n\x04name\x18\x05 \x01(\t:\xcc\x03\x92\x41\xc8\x03\n\xc5\x03J\xc2\x03{\"component\": {\"name\": \"engine\", \"purl\": \"pkg:github/scanoss/engine\", \"url\": \"https://github.com/scanoss/engine\", \"versions\": [{\"version\": \"5.0.0\", \"licenses\": [{\"name\": \"GNU General Public License v2.0\", \"spdx_id\": \"GPL-2.0\", \"is_spdx_approved\": true, \"url\": \"https://spdx.org/licenses/GPL-2.0.html\"}], \"date\": \"2024-01-15T10:30:00Z\"}], \"component\": \"engine\"}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Component versions successfully retrieved\"}}2\xca\x04\n\nComponents\x12o\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1e\x82\xd3\xe4\x93\x02\x18\"\x13/v2/components/echo:\x01*\x12\x8e\x01\n\x10SearchComponents\x12,.scanoss.api.components.v2.CompSearchRequest\x1a-.scanoss.api.components.v2.CompSearchResponse\"\x1d\x82\xd3\xe4\x93\x02\x17\x12\x15/v2/components/search\x12\x96\x01\n\x14GetComponentVersions\x12-.scanoss.api.components.v2.CompVersionRequest\x1a..scanoss.api.components.v2.CompVersionResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\x12\x17/v2/components/versions\x12\xa0\x01\n\x16GetComponentStatistics\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x36.scanoss.api.components.v2.ComponentsStatisticResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/v2/components/statistics:\x01*B\x8b\x03Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\x92\x41\xd0\x02\x12\x9d\x01\n\x1aSCANOSS Components Service\x12(Provides component intelligence services\"P\n\x12scanoss-components\x12%https://github.com/scanoss/components\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x02\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07Z8\n6\n\x07\x61pi_key\x12+\x08\x02\x12\x1a\x41PI key for authentication\x1a\tx-api-key \x02\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.components.v2.scanoss_components_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\337\002\022\235\001\n\032SCANOSS Components Service\022(Provides component intelligence services\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0\032\017api.scanoss.com*\002\002\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007Z8\n6\n\007api_key\022+\010\002\022\032API key for authentication\032\tx-api-key \002b\r\n\013\n\007api_key\022\000' + _globals['DESCRIPTOR']._serialized_options = b'Z5github.amrom.workers.dev/scanoss/papi/api/componentsv2;componentsv2\222A\320\002\022\235\001\n\032SCANOSS Components Service\022(Provides component intelligence services\"P\n\022scanoss-components\022%https://github.com/scanoss/components\032\023support@scanoss.com2\0032.0\032\017api.scanoss.com*\002\002\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007Z8\n6\n\007api_key\022+\010\002\022\032API key for authentication\032\tx-api-key \002' + _globals['_COMPSEARCHREQUEST']._loaded_options = None + _globals['_COMPSEARCHREQUEST']._serialized_options = b'\222A3\n1J/{\"search\": \"scanoss\", \"limit\": 10, \"offset\": 0}' + _globals['_COMPONENTSSTATISTICRESPONSE']._loaded_options = None + _globals['_COMPONENTSSTATISTICRESPONSE']._serialized_options = b'\222A\346\002\n\343\002J\340\002{\"component_statistics\": [{\"purl\": \"pkg:github/scanoss/engine@5.0.0\", \"version\": \"5.0.0\", \"statistics\": {\"total_source_files\": 156, \"total_lines\": 25430, \"total_blank_lines\": 3420, \"languages\": [{\"name\": \"C\", \"files\": 89}, {\"name\": \"C Header\", \"files\": 45}]}}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Component statistics successfully retrieved\"}}' + _globals['_COMPSEARCHRESPONSE_COMPONENT'].fields_by_name['component']._loaded_options = None + _globals['_COMPSEARCHRESPONSE_COMPONENT'].fields_by_name['component']._serialized_options = b'\030\001' + _globals['_COMPSEARCHRESPONSE']._loaded_options = None + _globals['_COMPSEARCHRESPONSE']._serialized_options = b'\222A\357\002\n\354\002J\351\002{\"components\": [{\"name\": \"scanoss-py\", \"purl\": \"pkg:github/scanoss/scanoss.py\", \"url\": \"https://github.com/scanoss/scanoss.py\", \"component\": \"scanoss-py\"}, {\"name\": \"engine\", \"purl\": \"pkg:github/scanoss/engine\", \"url\": \"https://github.com/scanoss/engine\", \"component\": \"engine\"}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Components successfully retrieved\"}}' + _globals['_COMPVERSIONREQUEST']._loaded_options = None + _globals['_COMPVERSIONREQUEST']._serialized_options = b'\222A6\n4J2{\"purl\": \"pkg:github/scanoss/engine\", \"limit\": 20}' + _globals['_COMPVERSIONRESPONSE_COMPONENT'].fields_by_name['component']._loaded_options = None + _globals['_COMPVERSIONRESPONSE_COMPONENT'].fields_by_name['component']._serialized_options = b'\030\001' + _globals['_COMPVERSIONRESPONSE']._loaded_options = None + _globals['_COMPVERSIONRESPONSE']._serialized_options = b'\222A\310\003\n\305\003J\302\003{\"component\": {\"name\": \"engine\", \"purl\": \"pkg:github/scanoss/engine\", \"url\": \"https://github.com/scanoss/engine\", \"versions\": [{\"version\": \"5.0.0\", \"licenses\": [{\"name\": \"GNU General Public License v2.0\", \"spdx_id\": \"GPL-2.0\", \"is_spdx_approved\": true, \"url\": \"https://spdx.org/licenses/GPL-2.0.html\"}], \"date\": \"2024-01-15T10:30:00Z\"}], \"component\": \"engine\"}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Component versions successfully retrieved\"}}' _globals['_COMPONENTS'].methods_by_name['Echo']._loaded_options = None - _globals['_COMPONENTS'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\022\027/api/v2/components/echo' + _globals['_COMPONENTS'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\030\"\023/v2/components/echo:\001*' _globals['_COMPONENTS'].methods_by_name['SearchComponents']._loaded_options = None - _globals['_COMPONENTS'].methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\033\022\031/api/v2/components/search' + _globals['_COMPONENTS'].methods_by_name['SearchComponents']._serialized_options = b'\202\323\344\223\002\027\022\025/v2/components/search' _globals['_COMPONENTS'].methods_by_name['GetComponentVersions']._loaded_options = None - _globals['_COMPONENTS'].methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002\035\022\033/api/v2/components/versions' + _globals['_COMPONENTS'].methods_by_name['GetComponentVersions']._serialized_options = b'\202\323\344\223\002\031\022\027/v2/components/versions' _globals['_COMPONENTS'].methods_by_name['GetComponentStatistics']._loaded_options = None - _globals['_COMPONENTS'].methods_by_name['GetComponentStatistics']._serialized_options = b'\202\323\344\223\002\"\"\035/api/v2/components/statistics:\001*' - _globals['_COMPSEARCHREQUEST']._serialized_start=203 - _globals['_COMPSEARCHREQUEST']._serialized_end=321 - _globals['_COMPSTATISTIC']._serialized_start=324 - _globals['_COMPSTATISTIC']._serialized_end=526 - _globals['_COMPSTATISTIC_LANGUAGE']._serialized_start=487 - _globals['_COMPSTATISTIC_LANGUAGE']._serialized_end=526 - _globals['_COMPSTATISTICRESPONSE']._serialized_start=529 - _globals['_COMPSTATISTICRESPONSE']._serialized_end=780 - _globals['_COMPSTATISTICRESPONSE_PURLS']._serialized_start=680 - _globals['_COMPSTATISTICRESPONSE_PURLS']._serialized_end=780 - _globals['_COMPSEARCHRESPONSE']._serialized_start=783 - _globals['_COMPSEARCHRESPONSE']._serialized_end=994 - _globals['_COMPSEARCHRESPONSE_COMPONENT']._serialized_start=937 - _globals['_COMPSEARCHRESPONSE_COMPONENT']._serialized_end=994 - _globals['_COMPVERSIONREQUEST']._serialized_start=996 - _globals['_COMPVERSIONREQUEST']._serialized_end=1045 - _globals['_COMPVERSIONRESPONSE']._serialized_start=1048 - _globals['_COMPVERSIONRESPONSE']._serialized_end=1532 - _globals['_COMPVERSIONRESPONSE_LICENSE']._serialized_start=1203 - _globals['_COMPVERSIONRESPONSE_LICENSE']._serialized_end=1282 - _globals['_COMPVERSIONRESPONSE_VERSION']._serialized_start=1284 - _globals['_COMPVERSIONRESPONSE_VERSION']._serialized_end=1398 - _globals['_COMPVERSIONRESPONSE_COMPONENT']._serialized_start=1401 - _globals['_COMPVERSIONRESPONSE_COMPONENT']._serialized_end=1532 - _globals['_COMPONENTS']._serialized_start=1535 - _globals['_COMPONENTS']._serialized_end=2122 + _globals['_COMPONENTS'].methods_by_name['GetComponentStatistics']._serialized_options = b'\202\323\344\223\002\036\"\031/v2/components/statistics:\001*' + _globals['_COMPSEARCHREQUEST']._serialized_start=204 + _globals['_COMPSEARCHREQUEST']._serialized_end=378 + _globals['_COMPSTATISTIC']._serialized_start=381 + _globals['_COMPSTATISTIC']._serialized_end=635 + _globals['_COMPSTATISTIC_LANGUAGE']._serialized_start=596 + _globals['_COMPSTATISTIC_LANGUAGE']._serialized_end=635 + _globals['_COMPONENTSSTATISTICRESPONSE']._serialized_start=638 + _globals['_COMPONENTSSTATISTICRESPONSE']._serialized_end=1331 + _globals['_COMPONENTSSTATISTICRESPONSE_COMPONENTSTATISTICS']._serialized_start=852 + _globals['_COMPONENTSSTATISTICRESPONSE_COMPONENTSTATISTICS']._serialized_end=966 + _globals['_COMPSEARCHRESPONSE']._serialized_start=1334 + _globals['_COMPSEARCHRESPONSE']._serialized_end=1937 + _globals['_COMPSEARCHRESPONSE_COMPONENT']._serialized_start=1488 + _globals['_COMPSEARCHRESPONSE_COMPONENT']._serialized_end=1563 + _globals['_COMPVERSIONREQUEST']._serialized_start=1939 + _globals['_COMPVERSIONREQUEST']._serialized_end=2047 + _globals['_COMPVERSIONRESPONSE']._serialized_start=2050 + _globals['_COMPVERSIONRESPONSE']._serialized_end=3042 + _globals['_COMPVERSIONRESPONSE_LICENSE']._serialized_start=2205 + _globals['_COMPVERSIONRESPONSE_LICENSE']._serialized_end=2311 + _globals['_COMPVERSIONRESPONSE_VERSION']._serialized_start=2313 + _globals['_COMPVERSIONRESPONSE_VERSION']._serialized_end=2427 + _globals['_COMPVERSIONRESPONSE_COMPONENT']._serialized_start=2430 + _globals['_COMPVERSIONRESPONSE_COMPONENT']._serialized_end=2579 + _globals['_COMPONENTS']._serialized_start=3045 + _globals['_COMPONENTS']._serialized_end=3631 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py index b80082b1..47490fbd 100644 --- a/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py +++ b/src/scanoss/api/components/v2/scanoss_components_pb2_grpc.py @@ -54,8 +54,8 @@ def __init__(self, channel): _registered_method=True) self.GetComponentStatistics = channel.unary_unary( '/scanoss.api.components.v2.Components/GetComponentStatistics', - request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.ComponentsStatisticResponse.FromString, _registered_method=True) @@ -112,8 +112,8 @@ def add_ComponentsServicer_to_server(servicer, server): ), 'GetComponentStatistics': grpc.unary_unary_rpc_method_handler( servicer.GetComponentStatistics, - request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, - response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.SerializeToString, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.ComponentsStatisticResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -224,8 +224,8 @@ def GetComponentStatistics(request, request, target, '/scanoss.api.components.v2.Components/GetComponentStatistics', - scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, - scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.CompStatisticResponse.FromString, + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_components_dot_v2_dot_scanoss__components__pb2.ComponentsStatisticResponse.FromString, options, channel_credentials, insecure, diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py index 6d488e19..06324447 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2.py @@ -27,50 +27,128 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"0\n\tAlgorithm\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\"\xf3\x01\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\x82\x02\n\x19\x41lgorithmsInRangeResponse\x12J\n\x05purls\x18\x01 \x03(\x0b\x32;.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\xe1\x01\n\x17VersionsInRangeResponse\x12H\n\x05purls\x18\x01 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.VersionsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x45\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x15\n\rversions_with\x18\x02 \x03(\t\x12\x18\n\x10versions_without\x18\x03 \x03(\t\"b\n\x04Hint\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08\x63\x61tegory\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0c\n\x04purl\x18\x06 \x01(\t\"\xe1\x01\n\rHintsResponse\x12?\n\x05purls\x18\x01 \x03(\x0b\x32\x30.scanoss.api.cryptography.v2.HintsResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\xee\x01\n\x14HintsInRangeResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.cryptography.v2.HintsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint2\x88\x07\n\x0c\x43ryptography\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/cryptography/echo:\x01*\x12\x8f\x01\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/cryptography/algorithms:\x01*\x12\xa5\x01\n\x14GetAlgorithmsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x36.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse\"1\x82\xd3\xe4\x93\x02+\"&/api/v2/cryptography/algorithmsInRange:\x01*\x12\x9f\x01\n\x12GetVersionsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x34.scanoss.api.cryptography.v2.VersionsInRangeResponse\"/\x82\xd3\xe4\x93\x02)\"$/api/v2/cryptography/versionsInRange:\x01*\x12\x96\x01\n\x0fGetHintsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.cryptography.v2.HintsInRangeResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/cryptography/hintsInRange:\x01*\x12\x8b\x01\n\x12GetEncryptionHints\x12\".scanoss.api.common.v2.PurlRequest\x1a*.scanoss.api.cryptography.v2.HintsResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/cryptography/hints:\x01*B\x9e\x02Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xdf\x01\x12y\n\x1cSCANOSS Cryptography Service\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/crpytography\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/cryptography/v2/scanoss-cryptography.proto\x12\x1bscanoss.api.cryptography.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"0\n\tAlgorithm\x12\x11\n\talgorithm\x18\x01 \x01(\t\x12\x10\n\x08strength\x18\x02 \x01(\t\"\xf7\x01\n\x11\x41lgorithmResponse\x12\x43\n\x05purls\x18\x01 \x03(\x0b\x32\x34.scanoss.api.cryptography.v2.AlgorithmResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm:\x02\x18\x01\"\x85\x01\n\x13\x43omponentAlgorithms\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x13\n\x0brequirement\x18\x03 \x01(\t\x12:\n\nalgorithms\x18\x04 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm\"\xe0\x04\n\x1c\x43omponentsAlgorithmsResponse\x12\x44\n\ncomponents\x18\x01 \x03(\x0b\x32\x30.scanoss.api.cryptography.v2.ComponentAlgorithms\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xc2\x03\x92\x41\xbe\x03\n\xbb\x03J\xb8\x03{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}, {\"algorithm\": \"RSA\", \"strength\": \"Strong\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"requirement\": \"~1.30.0\", \"version\": \"v1.30.0\", \"algorithms\": [{\"algorithm\": \"SHA-256\", \"strength\": \"Strong\"}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms Successfully retrieved\"}}\"\xc0\x03\n\x1b\x43omponentAlgorithmsResponse\x12\x43\n\tcomponent\x18\x01 \x01(\x0b\x32\x30.scanoss.api.cryptography.v2.ComponentAlgorithms\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xa4\x02\x92\x41\xa0\x02\n\x9d\x02J\x9a\x02{\"component\":{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}, {\"algorithm\": \"RSA\", \"strength\": \"Strong\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms Successfully retrieved\"}}\"\x86\x02\n\x19\x41lgorithmsInRangeResponse\x12J\n\x05purls\x18\x01 \x03(\x0b\x32;.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x62\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm:\x02\x18\x01\"\xa2\x05\n#ComponentsAlgorithmsInRangeResponse\x12^\n\ncomponents\x18\x01 \x03(\x0b\x32J.scanoss.api.cryptography.v2.ComponentsAlgorithmsInRangeResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1ag\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm:\xfa\x02\x92\x41\xf6\x02\n\xf3\x02J\xf0\x02{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"versions\": [\"1.0.0\", \"2.0.0\"], \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"versions\": [\"v1.30.0\"], \"algorithms\": [{\"algorithm\": \"SHA-256\", \"strength\": \"Strong\"}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms in range Successfully retrieved\"}}\"\xce\x04\n\"ComponentAlgorithmsInRangeResponse\x12\\\n\tcomponent\x18\x01 \x01(\x0b\x32I.scanoss.api.cryptography.v2.ComponentAlgorithmsInRangeResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1ag\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12:\n\nalgorithms\x18\x03 \x03(\x0b\x32&.scanoss.api.cryptography.v2.Algorithm:\xa9\x02\x92\x41\xa5\x02\n\xa2\x02J\x9f\x02{\"component\": {\"purl\": \"pkg:github/scanoss/engine\", \"versions\": [\"1.0.0\", \"2.0.0\", \"3.0.0\"], \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}, {\"algorithm\": \"RSA\", \"strength\": \"Strong\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms in range Successfully retrieved\"}}\"\x86\x02\n\x17VersionsInRangeResponse\x12H\n\x05purls\x18\x01 \x03(\x0b\x32\x39.scanoss.api.cryptography.v2.VersionsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x66\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12$\n\rversions_with\x18\x02 \x03(\tR\rversions_with\x12*\n\x10versions_without\x18\x03 \x03(\tR\x10versions_without:\x02\x18\x01\"\xeb\x04\n!ComponentsVersionsInRangeResponse\x12\\\n\ncomponents\x18\x01 \x03(\x0b\x32H.scanoss.api.cryptography.v2.ComponentsVersionsInRangeResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1ak\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12$\n\rversions_with\x18\x02 \x03(\tR\rversions_with\x12*\n\x10versions_without\x18\x03 \x03(\tR\x10versions_without:\xc3\x02\x92\x41\xbf\x02\n\xbc\x02J\xb9\x02{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"versions_with\": [\"2.0.0\", \"3.0.0\"], \"versions_without\": [\"1.0.0\"]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"versions_with\": [\"v1.30.0\"], \"versions_without\": [\"v1.29.0\"]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Version ranges Successfully retrieved\"}}\"\x8e\x04\n ComponentVersionsInRangeResponse\x12Z\n\tcomponent\x18\x01 \x01(\x0b\x32G.scanoss.api.cryptography.v2.ComponentVersionsInRangeResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1ak\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12$\n\rversions_with\x18\x02 \x03(\tR\rversions_with\x12*\n\x10versions_without\x18\x03 \x03(\tR\x10versions_without:\xe9\x01\x92\x41\xe5\x01\n\xe2\x01J\xdf\x01{\"component\": {\"purl\": \"pkg:github/scanoss/engine\", \"versions_with\": [\"2.0.0\", \"3.0.0\", \"4.0.0\"], \"versions_without\": [\"1.0.0\", \"1.5.0\"]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Version ranges Successfully retrieved\"}}\"b\n\x04Hint\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08\x63\x61tegory\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0c\n\x04purl\x18\x06 \x01(\t\"v\n\x0e\x43omponentHints\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x13\n\x0brequirement\x18\x03 \x01(\t\x12\x30\n\x05hints\x18\x04 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\xe1\x01\n\rHintsResponse\x12?\n\x05purls\x18\x01 \x03(\x0b\x32\x30.scanoss.api.cryptography.v2.HintsResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\xee\x01\n\x14HintsInRangeResponse\x12\x45\n\x05purls\x18\x01 \x03(\x0b\x32\x36.scanoss.api.cryptography.v2.HintsInRangeResponse.Purl\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aX\n\x04Purl\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\x91\x02\n\x1e\x43omponentsHintsInRangeResponse\x12Y\n\ncomponents\x18\x01 \x03(\x0b\x32\x45.scanoss.api.cryptography.v2.ComponentsHintsInRangeResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a]\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\x8e\x02\n\x1d\x43omponentHintsInRangeResponse\x12W\n\tcomponent\x18\x01 \x01(\x0b\x32\x44.scanoss.api.cryptography.v2.ComponentHintsInRangeResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a]\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x10\n\x08versions\x18\x02 \x03(\t\x12\x30\n\x05hints\x18\x03 \x03(\x0b\x32!.scanoss.api.cryptography.v2.Hint\"\xd4\x06\n!ComponentsEncryptionHintsResponse\x12?\n\ncomponents\x18\x01 \x03(\x0b\x32+.scanoss.api.cryptography.v2.ComponentHints\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xb6\x05\x92\x41\xb2\x05\n\xaf\x05J\xac\x05{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"hints\": [{\"id\": \"openssl-hint-001\", \"name\": \"OpenSSL\", \"description\": \"Industry standard cryptographic library\", \"category\": \"library\", \"url\": \"https://www.openssl.org/\", \"purl\": \"pkg:generic/openssl@3.0.0\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"requirement\": \"~1.30.0\", \"version\": \"v1.30.0\", \"hints\": [{\"id\": \"tls-protocol-001\", \"name\": \"TLS 1.3\", \"description\": \"Transport Layer Security protocol\", \"category\": \"protocol\", \"url\": \"https://tools.ietf.org/html/rfc8446\", \"purl\": \"\"}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Cryptographic hints Successfully retrieved\"}}\"\xb4\x04\n ComponentEncryptionHintsResponse\x12>\n\tcomponent\x18\x01 \x01(\x0b\x32+.scanoss.api.cryptography.v2.ComponentHints\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x98\x03\x92\x41\x94\x03\n\x91\x03J\x8e\x03{\"component\":{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"hints\": [{\"id\": \"openssl-hint-001\", \"name\": \"OpenSSL\", \"description\": \"Industry standard cryptographic library\", \"category\": \"library\", \"url\": \"https://www.openssl.org/\", \"purl\": \"pkg:generic/openssl@3.0.0\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Cryptographic hints Successfully retrieved\"}}2\x86\x14\n\x0c\x43ryptography\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/v2/cryptography/echo:\x01*\x12h\n\rGetAlgorithms\x12\".scanoss.api.common.v2.PurlRequest\x1a..scanoss.api.cryptography.v2.AlgorithmResponse\"\x03\x88\x02\x01\x12\xaa\x01\n\x16GetComponentAlgorithms\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x38.scanoss.api.cryptography.v2.ComponentAlgorithmsResponse\"-\x82\xd3\xe4\x93\x02\'\x12%/v2/cryptography/algorithms/component\x12\xb1\x01\n\x17GetComponentsAlgorithms\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x39.scanoss.api.cryptography.v2.ComponentsAlgorithmsResponse\"1\x82\xd3\xe4\x93\x02+\"&/v2/cryptography/algorithms/components:\x01*\x12w\n\x14GetAlgorithmsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x36.scanoss.api.cryptography.v2.AlgorithmsInRangeResponse\"\x03\x88\x02\x01\x12\xbe\x01\n\x1dGetComponentAlgorithmsInRange\x12\'.scanoss.api.common.v2.ComponentRequest\x1a?.scanoss.api.cryptography.v2.ComponentAlgorithmsInRangeResponse\"3\x82\xd3\xe4\x93\x02-\x12+/v2/cryptography/algorithms/range/component\x12\xc5\x01\n\x1eGetComponentsAlgorithmsInRange\x12(.scanoss.api.common.v2.ComponentsRequest\x1a@.scanoss.api.cryptography.v2.ComponentsAlgorithmsInRangeResponse\"7\x82\xd3\xe4\x93\x02\x31\",/v2/cryptography/algorithms/range/components:\x01*\x12s\n\x12GetVersionsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x34.scanoss.api.cryptography.v2.VersionsInRangeResponse\"\x03\x88\x02\x01\x12\xc3\x01\n\x1bGetComponentVersionsInRange\x12\'.scanoss.api.common.v2.ComponentRequest\x1a=.scanoss.api.cryptography.v2.ComponentVersionsInRangeResponse\"<\x82\xd3\xe4\x93\x02\x36\x12\x34/v2/cryptography/algorithms/versions/range/component\x12\xca\x01\n\x1cGetComponentsVersionsInRange\x12(.scanoss.api.common.v2.ComponentsRequest\x1a>.scanoss.api.cryptography.v2.ComponentsVersionsInRangeResponse\"@\x82\xd3\xe4\x93\x02:\"5/v2/cryptography/algorithms/versions/range/components:\x01*\x12m\n\x0fGetHintsInRange\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.cryptography.v2.HintsInRangeResponse\"\x03\x88\x02\x01\x12\xaf\x01\n\x18GetComponentHintsInRange\x12\'.scanoss.api.common.v2.ComponentRequest\x1a:.scanoss.api.cryptography.v2.ComponentHintsInRangeResponse\".\x82\xd3\xe4\x93\x02(\x12&/v2/cryptography/hints/range/component\x12\xb6\x01\n\x19GetComponentsHintsInRange\x12(.scanoss.api.common.v2.ComponentsRequest\x1a;.scanoss.api.cryptography.v2.ComponentsHintsInRangeResponse\"2\x82\xd3\xe4\x93\x02,\"\'/v2/cryptography/hints/range/components:\x01*\x12i\n\x12GetEncryptionHints\x12\".scanoss.api.common.v2.PurlRequest\x1a*.scanoss.api.cryptography.v2.HintsResponse\"\x03\x88\x02\x01\x12\xaf\x01\n\x1bGetComponentEncryptionHints\x12\'.scanoss.api.common.v2.ComponentRequest\x1a=.scanoss.api.cryptography.v2.ComponentEncryptionHintsResponse\"(\x82\xd3\xe4\x93\x02\"\x12 /v2/cryptography/hints/component\x12\xb6\x01\n\x1cGetComponentsEncryptionHints\x12(.scanoss.api.common.v2.ComponentsRequest\x1a>.scanoss.api.cryptography.v2.ComponentsEncryptionHintsResponse\",\x82\xd3\xe4\x93\x02&\"!/v2/cryptography/hints/components:\x01*B\xbb\x03Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\x92\x41\xfc\x02\x12\x83\x02\n\x1cSCANOSS Cryptography Service\x12\x87\x01\x43ryptography service provides cryptographic intelligence for software components including algorithm detection and encryption analysis.\"T\n\x14scanoss-cryptography\x12\'https://github.com/scanoss/cryptography\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x01\x02\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scanoss.api.cryptography.v2.scanoss_cryptography_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\337\001\022y\n\034SCANOSS Cryptography Service\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/crpytography\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['DESCRIPTOR']._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/cryptographyv2;cryptographyv2\222A\374\002\022\203\002\n\034SCANOSS Cryptography Service\022\207\001Cryptography service provides cryptographic intelligence for software components including algorithm detection and encryption analysis.\"T\n\024scanoss-cryptography\022\'https://github.com/scanoss/cryptography\032\023support@scanoss.com2\0032.0\032\017api.scanoss.com*\002\001\0022\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_ALGORITHMRESPONSE']._loaded_options = None + _globals['_ALGORITHMRESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTSALGORITHMSRESPONSE']._loaded_options = None + _globals['_COMPONENTSALGORITHMSRESPONSE']._serialized_options = b'\222A\276\003\n\273\003J\270\003{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}, {\"algorithm\": \"RSA\", \"strength\": \"Strong\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"requirement\": \"~1.30.0\", \"version\": \"v1.30.0\", \"algorithms\": [{\"algorithm\": \"SHA-256\", \"strength\": \"Strong\"}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms Successfully retrieved\"}}' + _globals['_COMPONENTALGORITHMSRESPONSE']._loaded_options = None + _globals['_COMPONENTALGORITHMSRESPONSE']._serialized_options = b'\222A\240\002\n\235\002J\232\002{\"component\":{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}, {\"algorithm\": \"RSA\", \"strength\": \"Strong\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms Successfully retrieved\"}}' + _globals['_ALGORITHMSINRANGERESPONSE']._loaded_options = None + _globals['_ALGORITHMSINRANGERESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTSALGORITHMSINRANGERESPONSE']._loaded_options = None + _globals['_COMPONENTSALGORITHMSINRANGERESPONSE']._serialized_options = b'\222A\366\002\n\363\002J\360\002{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"versions\": [\"1.0.0\", \"2.0.0\"], \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"versions\": [\"v1.30.0\"], \"algorithms\": [{\"algorithm\": \"SHA-256\", \"strength\": \"Strong\"}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms in range Successfully retrieved\"}}' + _globals['_COMPONENTALGORITHMSINRANGERESPONSE']._loaded_options = None + _globals['_COMPONENTALGORITHMSINRANGERESPONSE']._serialized_options = b'\222A\245\002\n\242\002J\237\002{\"component\": {\"purl\": \"pkg:github/scanoss/engine\", \"versions\": [\"1.0.0\", \"2.0.0\", \"3.0.0\"], \"algorithms\": [{\"algorithm\": \"AES\", \"strength\": \"Strong\"}, {\"algorithm\": \"RSA\", \"strength\": \"Strong\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Algorithms in range Successfully retrieved\"}}' + _globals['_VERSIONSINRANGERESPONSE']._loaded_options = None + _globals['_VERSIONSINRANGERESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTSVERSIONSINRANGERESPONSE']._loaded_options = None + _globals['_COMPONENTSVERSIONSINRANGERESPONSE']._serialized_options = b'\222A\277\002\n\274\002J\271\002{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"versions_with\": [\"2.0.0\", \"3.0.0\"], \"versions_without\": [\"1.0.0\"]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"versions_with\": [\"v1.30.0\"], \"versions_without\": [\"v1.29.0\"]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Version ranges Successfully retrieved\"}}' + _globals['_COMPONENTVERSIONSINRANGERESPONSE']._loaded_options = None + _globals['_COMPONENTVERSIONSINRANGERESPONSE']._serialized_options = b'\222A\345\001\n\342\001J\337\001{\"component\": {\"purl\": \"pkg:github/scanoss/engine\", \"versions_with\": [\"2.0.0\", \"3.0.0\", \"4.0.0\"], \"versions_without\": [\"1.0.0\", \"1.5.0\"]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Version ranges Successfully retrieved\"}}' + _globals['_COMPONENTSENCRYPTIONHINTSRESPONSE']._loaded_options = None + _globals['_COMPONENTSENCRYPTIONHINTSRESPONSE']._serialized_options = b'\222A\262\005\n\257\005J\254\005{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"hints\": [{\"id\": \"openssl-hint-001\", \"name\": \"OpenSSL\", \"description\": \"Industry standard cryptographic library\", \"category\": \"library\", \"url\": \"https://www.openssl.org/\", \"purl\": \"pkg:generic/openssl@3.0.0\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\", \"requirement\": \"~1.30.0\", \"version\": \"v1.30.0\", \"hints\": [{\"id\": \"tls-protocol-001\", \"name\": \"TLS 1.3\", \"description\": \"Transport Layer Security protocol\", \"category\": \"protocol\", \"url\": \"https://tools.ietf.org/html/rfc8446\", \"purl\": \"\"}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Cryptographic hints Successfully retrieved\"}}' + _globals['_COMPONENTENCRYPTIONHINTSRESPONSE']._loaded_options = None + _globals['_COMPONENTENCRYPTIONHINTSRESPONSE']._serialized_options = b'\222A\224\003\n\221\003J\216\003{\"component\":{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \">=5.0.0\", \"version\": \"5.0.0\", \"hints\": [{\"id\": \"openssl-hint-001\", \"name\": \"OpenSSL\", \"description\": \"Industry standard cryptographic library\", \"category\": \"library\", \"url\": \"https://www.openssl.org/\", \"purl\": \"pkg:generic/openssl@3.0.0\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Cryptographic hints Successfully retrieved\"}}' _globals['_CRYPTOGRAPHY'].methods_by_name['Echo']._loaded_options = None - _globals['_CRYPTOGRAPHY'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/cryptography/echo:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/v2/cryptography/echo:\001*' _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithms']._loaded_options = None - _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithms']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/cryptography/algorithms:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithms']._serialized_options = b'\210\002\001' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentAlgorithms']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentAlgorithms']._serialized_options = b'\202\323\344\223\002\'\022%/v2/cryptography/algorithms/component' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsAlgorithms']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsAlgorithms']._serialized_options = b'\202\323\344\223\002+\"&/v2/cryptography/algorithms/components:\001*' _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithmsInRange']._loaded_options = None - _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithmsInRange']._serialized_options = b'\202\323\344\223\002+\"&/api/v2/cryptography/algorithmsInRange:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetAlgorithmsInRange']._serialized_options = b'\210\002\001' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentAlgorithmsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentAlgorithmsInRange']._serialized_options = b'\202\323\344\223\002-\022+/v2/cryptography/algorithms/range/component' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsAlgorithmsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsAlgorithmsInRange']._serialized_options = b'\202\323\344\223\0021\",/v2/cryptography/algorithms/range/components:\001*' _globals['_CRYPTOGRAPHY'].methods_by_name['GetVersionsInRange']._loaded_options = None - _globals['_CRYPTOGRAPHY'].methods_by_name['GetVersionsInRange']._serialized_options = b'\202\323\344\223\002)\"$/api/v2/cryptography/versionsInRange:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetVersionsInRange']._serialized_options = b'\210\002\001' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentVersionsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentVersionsInRange']._serialized_options = b'\202\323\344\223\0026\0224/v2/cryptography/algorithms/versions/range/component' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsVersionsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsVersionsInRange']._serialized_options = b'\202\323\344\223\002:\"5/v2/cryptography/algorithms/versions/range/components:\001*' _globals['_CRYPTOGRAPHY'].methods_by_name['GetHintsInRange']._loaded_options = None - _globals['_CRYPTOGRAPHY'].methods_by_name['GetHintsInRange']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/cryptography/hintsInRange:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetHintsInRange']._serialized_options = b'\210\002\001' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentHintsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentHintsInRange']._serialized_options = b'\202\323\344\223\002(\022&/v2/cryptography/hints/range/component' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsHintsInRange']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsHintsInRange']._serialized_options = b'\202\323\344\223\002,\"\'/v2/cryptography/hints/range/components:\001*' _globals['_CRYPTOGRAPHY'].methods_by_name['GetEncryptionHints']._loaded_options = None - _globals['_CRYPTOGRAPHY'].methods_by_name['GetEncryptionHints']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/cryptography/hints:\001*' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetEncryptionHints']._serialized_options = b'\210\002\001' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentEncryptionHints']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentEncryptionHints']._serialized_options = b'\202\323\344\223\002\"\022 /v2/cryptography/hints/component' + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsEncryptionHints']._loaded_options = None + _globals['_CRYPTOGRAPHY'].methods_by_name['GetComponentsEncryptionHints']._serialized_options = b'\202\323\344\223\002&\"!/v2/cryptography/hints/components:\001*' _globals['_ALGORITHM']._serialized_start=209 _globals['_ALGORITHM']._serialized_end=257 _globals['_ALGORITHMRESPONSE']._serialized_start=260 - _globals['_ALGORITHMRESPONSE']._serialized_end=503 + _globals['_ALGORITHMRESPONSE']._serialized_end=507 _globals['_ALGORITHMRESPONSE_PURLS']._serialized_start=405 _globals['_ALGORITHMRESPONSE_PURLS']._serialized_end=503 - _globals['_ALGORITHMSINRANGERESPONSE']._serialized_start=506 - _globals['_ALGORITHMSINRANGERESPONSE']._serialized_end=764 - _globals['_ALGORITHMSINRANGERESPONSE_PURL']._serialized_start=666 - _globals['_ALGORITHMSINRANGERESPONSE_PURL']._serialized_end=764 - _globals['_VERSIONSINRANGERESPONSE']._serialized_start=767 - _globals['_VERSIONSINRANGERESPONSE']._serialized_end=992 - _globals['_VERSIONSINRANGERESPONSE_PURL']._serialized_start=923 - _globals['_VERSIONSINRANGERESPONSE_PURL']._serialized_end=992 - _globals['_HINT']._serialized_start=994 - _globals['_HINT']._serialized_end=1092 - _globals['_HINTSRESPONSE']._serialized_start=1095 - _globals['_HINTSRESPONSE']._serialized_end=1320 - _globals['_HINTSRESPONSE_PURLS']._serialized_start=1232 - _globals['_HINTSRESPONSE_PURLS']._serialized_end=1320 - _globals['_HINTSINRANGERESPONSE']._serialized_start=1323 - _globals['_HINTSINRANGERESPONSE']._serialized_end=1561 - _globals['_HINTSINRANGERESPONSE_PURL']._serialized_start=1473 - _globals['_HINTSINRANGERESPONSE_PURL']._serialized_end=1561 - _globals['_CRYPTOGRAPHY']._serialized_start=1564 - _globals['_CRYPTOGRAPHY']._serialized_end=2468 + _globals['_COMPONENTALGORITHMS']._serialized_start=510 + _globals['_COMPONENTALGORITHMS']._serialized_end=643 + _globals['_COMPONENTSALGORITHMSRESPONSE']._serialized_start=646 + _globals['_COMPONENTSALGORITHMSRESPONSE']._serialized_end=1254 + _globals['_COMPONENTALGORITHMSRESPONSE']._serialized_start=1257 + _globals['_COMPONENTALGORITHMSRESPONSE']._serialized_end=1705 + _globals['_ALGORITHMSINRANGERESPONSE']._serialized_start=1708 + _globals['_ALGORITHMSINRANGERESPONSE']._serialized_end=1970 + _globals['_ALGORITHMSINRANGERESPONSE_PURL']._serialized_start=1868 + _globals['_ALGORITHMSINRANGERESPONSE_PURL']._serialized_end=1966 + _globals['_COMPONENTSALGORITHMSINRANGERESPONSE']._serialized_start=1973 + _globals['_COMPONENTSALGORITHMSINRANGERESPONSE']._serialized_end=2647 + _globals['_COMPONENTSALGORITHMSINRANGERESPONSE_COMPONENT']._serialized_start=2163 + _globals['_COMPONENTSALGORITHMSINRANGERESPONSE_COMPONENT']._serialized_end=2266 + _globals['_COMPONENTALGORITHMSINRANGERESPONSE']._serialized_start=2650 + _globals['_COMPONENTALGORITHMSINRANGERESPONSE']._serialized_end=3240 + _globals['_COMPONENTALGORITHMSINRANGERESPONSE_COMPONENT']._serialized_start=2163 + _globals['_COMPONENTALGORITHMSINRANGERESPONSE_COMPONENT']._serialized_end=2266 + _globals['_VERSIONSINRANGERESPONSE']._serialized_start=3243 + _globals['_VERSIONSINRANGERESPONSE']._serialized_end=3505 + _globals['_VERSIONSINRANGERESPONSE_PURL']._serialized_start=3399 + _globals['_VERSIONSINRANGERESPONSE_PURL']._serialized_end=3501 + _globals['_COMPONENTSVERSIONSINRANGERESPONSE']._serialized_start=3508 + _globals['_COMPONENTSVERSIONSINRANGERESPONSE']._serialized_end=4127 + _globals['_COMPONENTSVERSIONSINRANGERESPONSE_COMPONENT']._serialized_start=3694 + _globals['_COMPONENTSVERSIONSINRANGERESPONSE_COMPONENT']._serialized_end=3801 + _globals['_COMPONENTVERSIONSINRANGERESPONSE']._serialized_start=4130 + _globals['_COMPONENTVERSIONSINRANGERESPONSE']._serialized_end=4656 + _globals['_COMPONENTVERSIONSINRANGERESPONSE_COMPONENT']._serialized_start=3694 + _globals['_COMPONENTVERSIONSINRANGERESPONSE_COMPONENT']._serialized_end=3801 + _globals['_HINT']._serialized_start=4658 + _globals['_HINT']._serialized_end=4756 + _globals['_COMPONENTHINTS']._serialized_start=4758 + _globals['_COMPONENTHINTS']._serialized_end=4876 + _globals['_HINTSRESPONSE']._serialized_start=4879 + _globals['_HINTSRESPONSE']._serialized_end=5104 + _globals['_HINTSRESPONSE_PURLS']._serialized_start=5016 + _globals['_HINTSRESPONSE_PURLS']._serialized_end=5104 + _globals['_HINTSINRANGERESPONSE']._serialized_start=5107 + _globals['_HINTSINRANGERESPONSE']._serialized_end=5345 + _globals['_HINTSINRANGERESPONSE_PURL']._serialized_start=5257 + _globals['_HINTSINRANGERESPONSE_PURL']._serialized_end=5345 + _globals['_COMPONENTSHINTSINRANGERESPONSE']._serialized_start=5348 + _globals['_COMPONENTSHINTSINRANGERESPONSE']._serialized_end=5621 + _globals['_COMPONENTSHINTSINRANGERESPONSE_COMPONENT']._serialized_start=5528 + _globals['_COMPONENTSHINTSINRANGERESPONSE_COMPONENT']._serialized_end=5621 + _globals['_COMPONENTHINTSINRANGERESPONSE']._serialized_start=5624 + _globals['_COMPONENTHINTSINRANGERESPONSE']._serialized_end=5894 + _globals['_COMPONENTHINTSINRANGERESPONSE_COMPONENT']._serialized_start=5528 + _globals['_COMPONENTHINTSINRANGERESPONSE_COMPONENT']._serialized_end=5621 + _globals['_COMPONENTSENCRYPTIONHINTSRESPONSE']._serialized_start=5897 + _globals['_COMPONENTSENCRYPTIONHINTSRESPONSE']._serialized_end=6749 + _globals['_COMPONENTENCRYPTIONHINTSRESPONSE']._serialized_start=6752 + _globals['_COMPONENTENCRYPTIONHINTSRESPONSE']._serialized_end=7316 + _globals['_CRYPTOGRAPHY']._serialized_start=7319 + _globals['_CRYPTOGRAPHY']._serialized_end=9885 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py index e015e875..5aa7f6ad 100644 --- a/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py +++ b/src/scanoss/api/cryptography/v2/scanoss_cryptography_pb2_grpc.py @@ -28,7 +28,10 @@ class CryptographyStub(object): """ - Expose all of the SCANOSS Cryptography RPCs here + Cryptography Service Definition + + Provides comprehensive cryptographic intelligence for software components + including algorithm detection, encryption hints, and security assessments. """ def __init__(self, channel): @@ -47,70 +50,280 @@ def __init__(self, channel): request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.FromString, _registered_method=True) + self.GetComponentAlgorithms = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentAlgorithms', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentAlgorithmsResponse.FromString, + _registered_method=True) + self.GetComponentsAlgorithms = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsAlgorithms', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsAlgorithmsResponse.FromString, + _registered_method=True) self.GetAlgorithmsInRange = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetAlgorithmsInRange', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmsInRangeResponse.FromString, _registered_method=True) + self.GetComponentAlgorithmsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentAlgorithmsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentAlgorithmsInRangeResponse.FromString, + _registered_method=True) + self.GetComponentsAlgorithmsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsAlgorithmsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsAlgorithmsInRangeResponse.FromString, + _registered_method=True) self.GetVersionsInRange = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetVersionsInRange', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.VersionsInRangeResponse.FromString, _registered_method=True) + self.GetComponentVersionsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentVersionsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentVersionsInRangeResponse.FromString, + _registered_method=True) + self.GetComponentsVersionsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsVersionsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsVersionsInRangeResponse.FromString, + _registered_method=True) self.GetHintsInRange = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetHintsInRange', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsInRangeResponse.FromString, _registered_method=True) + self.GetComponentHintsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentHintsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentHintsInRangeResponse.FromString, + _registered_method=True) + self.GetComponentsHintsInRange = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsHintsInRange', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsHintsInRangeResponse.FromString, + _registered_method=True) self.GetEncryptionHints = channel.unary_unary( '/scanoss.api.cryptography.v2.Cryptography/GetEncryptionHints', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsResponse.FromString, _registered_method=True) + self.GetComponentEncryptionHints = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentEncryptionHints', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentEncryptionHintsResponse.FromString, + _registered_method=True) + self.GetComponentsEncryptionHints = channel.unary_unary( + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsEncryptionHints', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsEncryptionHintsResponse.FromString, + _registered_method=True) class CryptographyServicer(object): """ - Expose all of the SCANOSS Cryptography RPCs here + Cryptography Service Definition + + Provides comprehensive cryptographic intelligence for software components + including algorithm detection, encryption hints, and security assessments. """ def Echo(self, request, context): - """Standard echo + """ + Returns the same message that was sent, used for health checks and connectivity testing """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetAlgorithms(self, request, context): - """Get Cryptographic algorithms associated with a list of PURLs and, optionally, a requirement + """****** Algorithms ****** // + + + Get cryptographic algorithms associated with a list of PURLs - legacy endpoint. + + Legacy method for retrieving cryptographic algorithms used by software components. + Use GetComponentAlgorithms or GetComponentsAlgorithms instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentAlgorithms(self, request, context): + """ + Get cryptographic algorithms associated with a single software component. + + Analyzes the component and returns cryptographic algorithms detected in the codebase + including algorithm names and strength classifications. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#getcomponentalgorithms + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsAlgorithms(self, request, context): + """ + Get cryptographic algorithms associated with multiple software components in a single request. + + Analyzes multiple components and returns cryptographic algorithms detected in each codebase + including algorithm names and strength classifications. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#getcomponentsalgorithms """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetAlgorithmsInRange(self, request, context): - """Given a list of PURLS and version ranges, get a list of cryptographic algorithms used + """****** Algorithms in range ****** // + + + Get cryptographic algorithms used across version ranges - legacy endpoint. + + Legacy method for retrieving cryptographic algorithms across component version ranges. + Use GetComponentAlgorithmsInRange or GetComponentsAlgorithmsInRange instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentAlgorithmsInRange(self, request, context): + """ + Get cryptographic algorithms used by a component across specified version ranges. + + Analyzes the component across version ranges and returns all cryptographic algorithms + detected along with the versions where they appear. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#getcomponentalgorithmsinrange + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsAlgorithmsInRange(self, request, context): + """ + Get cryptographic algorithms used by multiple components across specified version ranges. + + Analyzes multiple components across version ranges and returns all cryptographic algorithms + detected along with the versions where they appear for each component. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#getcomponentsalgorithmsinrange """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetVersionsInRange(self, request, context): - """Given a list of PURLS and version ranges, get a list of versions that do/do not contain cryptographic algorithms + """****** Versions in range ****** // + + + Get component versions that contain or don't contain cryptographic algorithms - legacy endpoint. + + Legacy method for retrieving version lists based on cryptographic algorithm presence. + Use ComponentVersionsInRange or ComponentsVersionsInRange instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentVersionsInRange(self, request, context): + """ + Get component versions that contain or don't contain cryptographic algorithms within specified ranges. + + Returns lists of versions that either contain cryptographic algorithms or don't, + helping assess cryptographic presence across component evolution. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#componentversionsinrange + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsVersionsInRange(self, request, context): + """ + Get multiple component versions that contain or don't contain cryptographic algorithms within specified ranges. + + Returns lists of versions for multiple components that either contain cryptographic algorithms or don't, + helping assess cryptographic presence across component evolution in batch operations. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#componentsversionsinrange """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetHintsInRange(self, request, context): - """Given a list of PURLS and version ranges, get hints related to protocol/library/sdk/framework + """****** Hints in range ****** // + + + Get cryptographic hints across version ranges - legacy endpoint. + + Legacy method for retrieving cryptographic hints related to protocols, libraries, + SDKs and frameworks across version ranges. Use ComponentHintsInRange or ComponentsHintsInRange instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentHintsInRange(self, request, context): + """ + Get cryptographic hints across version ranges - legacy endpoint. + + Legacy method for retrieving cryptographic hints related to protocols, libraries, + SDKs and frameworks across version ranges. Use ComponentHintsInRange or ComponentsHintsInRange instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsHintsInRange(self, request, context): + """ + Get cryptographic hints across version ranges - legacy endpoint. + + Legacy method for retrieving cryptographic hints related to protocols, libraries, + SDKs and frameworks across version ranges. Use ComponentHintsInRange or ComponentsHintsInRange instead. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetEncryptionHints(self, request, context): - """Given a list of PURLS, get hints related to protocol/library/sdk/framework + """****** Encryption hints ****** // + + + Get encryption hints for components - legacy endpoint. + + Legacy method for retrieving hints about cryptographic protocols, libraries, + SDKs and frameworks used by components. Use ComponentHintsInRange or ComponentsHintsInRange instead. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentEncryptionHints(self, request, context): + """ + Get cryptographic hints for a single component. + + Returns hints about cryptographic protocols, libraries, SDKs and frameworks + used by the component, providing insights into cryptographic dependencies. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#componenthintsinrange + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsEncryptionHints(self, request, context): + """ + Get cryptographic hints for multiple components in a single request. + + Returns hints about cryptographic protocols, libraries, SDKs and frameworks + used by multiple components, providing insights into cryptographic dependencies. + + See: https://github.com/scanoss/papi/blob/main/protobuf/scanoss/api/cryptography/v2/README.md#componentshintsinrange """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -129,26 +342,76 @@ def add_CryptographyServicer_to_server(servicer, server): request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmResponse.SerializeToString, ), + 'GetComponentAlgorithms': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentAlgorithms, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentAlgorithmsResponse.SerializeToString, + ), + 'GetComponentsAlgorithms': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsAlgorithms, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsAlgorithmsResponse.SerializeToString, + ), 'GetAlgorithmsInRange': grpc.unary_unary_rpc_method_handler( servicer.GetAlgorithmsInRange, request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.AlgorithmsInRangeResponse.SerializeToString, ), + 'GetComponentAlgorithmsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentAlgorithmsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentAlgorithmsInRangeResponse.SerializeToString, + ), + 'GetComponentsAlgorithmsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsAlgorithmsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsAlgorithmsInRangeResponse.SerializeToString, + ), 'GetVersionsInRange': grpc.unary_unary_rpc_method_handler( servicer.GetVersionsInRange, request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.VersionsInRangeResponse.SerializeToString, ), + 'GetComponentVersionsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentVersionsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentVersionsInRangeResponse.SerializeToString, + ), + 'GetComponentsVersionsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsVersionsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsVersionsInRangeResponse.SerializeToString, + ), 'GetHintsInRange': grpc.unary_unary_rpc_method_handler( servicer.GetHintsInRange, request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsInRangeResponse.SerializeToString, ), + 'GetComponentHintsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentHintsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentHintsInRangeResponse.SerializeToString, + ), + 'GetComponentsHintsInRange': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsHintsInRange, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsHintsInRangeResponse.SerializeToString, + ), 'GetEncryptionHints': grpc.unary_unary_rpc_method_handler( servicer.GetEncryptionHints, request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.HintsResponse.SerializeToString, ), + 'GetComponentEncryptionHints': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentEncryptionHints, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentEncryptionHintsResponse.SerializeToString, + ), + 'GetComponentsEncryptionHints': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsEncryptionHints, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsEncryptionHintsResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.cryptography.v2.Cryptography', rpc_method_handlers) @@ -159,7 +422,10 @@ def add_CryptographyServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class Cryptography(object): """ - Expose all of the SCANOSS Cryptography RPCs here + Cryptography Service Definition + + Provides comprehensive cryptographic intelligence for software components + including algorithm detection, encryption hints, and security assessments. """ @staticmethod @@ -216,6 +482,60 @@ def GetAlgorithms(request, metadata, _registered_method=True) + @staticmethod + def GetComponentAlgorithms(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentAlgorithms', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentAlgorithmsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsAlgorithms(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsAlgorithms', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsAlgorithmsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def GetAlgorithmsInRange(request, target, @@ -243,6 +563,60 @@ def GetAlgorithmsInRange(request, metadata, _registered_method=True) + @staticmethod + def GetComponentAlgorithmsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentAlgorithmsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentAlgorithmsInRangeResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsAlgorithmsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsAlgorithmsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsAlgorithmsInRangeResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def GetVersionsInRange(request, target, @@ -270,6 +644,60 @@ def GetVersionsInRange(request, metadata, _registered_method=True) + @staticmethod + def GetComponentVersionsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentVersionsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentVersionsInRangeResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsVersionsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsVersionsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsVersionsInRangeResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def GetHintsInRange(request, target, @@ -297,6 +725,60 @@ def GetHintsInRange(request, metadata, _registered_method=True) + @staticmethod + def GetComponentHintsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentHintsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentHintsInRangeResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsHintsInRange(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsHintsInRange', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsHintsInRangeResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def GetEncryptionHints(request, target, @@ -323,3 +805,57 @@ def GetEncryptionHints(request, timeout, metadata, _registered_method=True) + + @staticmethod + def GetComponentEncryptionHints(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentEncryptionHints', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentEncryptionHintsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentsEncryptionHints(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.cryptography.v2.Cryptography/GetComponentsEncryptionHints', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_cryptography_dot_v2_dot_scanoss__cryptography__pb2.ComponentsEncryptionHintsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py index 59399d84..37662af9 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xef\x01\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls\"\x98\x04\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07spdx_id\x18\x02 \x01(\t\x12\x18\n\x10is_spdx_approved\x18\x03 \x01(\x08\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies\"z\n\x1bTransitiveDependencyRequest\x12\x11\n\tecosystem\x18\x01 \x01(\t\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x12\r\n\x05limit\x18\x03 \x01(\x05\x12*\n\x05purls\x18\x05 \x03(\x0b\x32\x1b.scanoss.api.common.v2.Purl\"\xe2\x01\n\x1cTransitiveDependencyResponse\x12\\\n\x0c\x64\x65pendencies\x18\x01 \x03(\x0b\x32\x46.scanoss.api.dependencies.v2.TransitiveDependencyResponse.Dependencies\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a-\n\x0c\x44\x65pendencies\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t2\xe7\x03\n\x0c\x44\x65pendencies\x12u\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/dependencies/echo:\x01*\x12\xa0\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\",\x82\xd3\xe4\x93\x02&\"!/api/v2/dependencies/dependencies:\x01*\x12\xbc\x01\n\x19GetTransitiveDependencies\x12\x38.scanoss.api.dependencies.v2.TransitiveDependencyRequest\x1a\x39.scanoss.api.dependencies.v2.TransitiveDependencyResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/dependencies/transitive:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6scanoss/api/dependencies/v2/scanoss-dependencies.proto\x12\x1bscanoss.api.dependencies.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xda\x02\n\x11\x44\x65pendencyRequest\x12\x43\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Files\x12\r\n\x05\x64\x65pth\x18\x02 \x01(\x05\x1a*\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x1aZ\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\x43\n\x05purls\x18\x02 \x03(\x0b\x32\x34.scanoss.api.dependencies.v2.DependencyRequest.Purls:i\x18\x01\x92\x41\x64\nbJ`{\"files\":[{\"file\":\"package.json\",\"purls\":[{\"purl\":\"pkg:npm/express\",\"requirement\":\"^4.18.0\"}]}]}\"\xd8\x07\n\x12\x44\x65pendencyResponse\x12\x44\n\x05\x66iles\x18\x01 \x03(\x0b\x32\x35.scanoss.api.dependencies.v2.DependencyResponse.Files\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1ak\n\x08Licenses\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x18\n\x07spdx_id\x18\x02 \x01(\tR\x07spdx_id\x12*\n\x10is_spdx_approved\x18\x03 \x01(\x08R\x10is_spdx_approved\x12\x0b\n\x03url\x18\x04 \x01(\t\x1a\xaa\x01\n\x0c\x44\x65pendencies\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x0c\n\x04purl\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12J\n\x08licenses\x18\x04 \x03(\x0b\x32\x38.scanoss.api.dependencies.v2.DependencyResponse.Licenses\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x0f\n\x07\x63omment\x18\x06 \x01(\t\x1a\x85\x01\n\x05\x46iles\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12R\n\x0c\x64\x65pendencies\x18\x04 \x03(\x0b\x32<.scanoss.api.dependencies.v2.DependencyResponse.Dependencies:\xa2\x03\x18\x01\x92\x41\x9c\x03\n\x99\x03J\x96\x03{\"files\":[{\"file\":\"package.json\",\"id\":\"dependency\",\"status\":\"pending\",\"dependencies\":[{\"component\":\"express\",\"purl\":\"pkg:npm/express\",\"version\":\"4.18.2\",\"licenses\":[{\"name\":\"MIT\",\"spdx_id\":\"MIT\",\"is_spdx_approved\":true,\"url\":\"https://opensource.org/licenses/MIT\"}],\"url\":\"https://www.npmjs.com/package/express\",\"comment\":\"\"}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Dependencies successfully retrieved\"}}\"\x8d\x02\n\x1bTransitiveDependencyRequest\x12\r\n\x05\x64\x65pth\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\x12;\n\ncomponents\x18\x03 \x03(\x0b\x32\'.scanoss.api.common.v2.ComponentRequest:\x92\x01\x92\x41\x8e\x01\n\x8b\x01J\x88\x01{\"depth\":3,\"limit\":50,\"components\":[{\"purl\":\"pkg:npm/express\",\"requirement\":\"4.18.0\"},{\"purl\":\"pkg:npm/lodash\",\"requirement\":\"4.17.0\"}]}\"\x89\x04\n\x1cTransitiveDependencyResponse\x12Y\n\x0c\x64\x65pendencies\x18\x01 \x03(\x0b\x32\x43.scanoss.api.dependencies.v2.TransitiveDependencyResponse.Component\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a?\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x13\n\x0brequirement\x18\x03 \x01(\t:\x95\x02\x92\x41\x91\x02\n\x8e\x02J\x8b\x02{\"dependencies\":[{\"purl\":\"pkg:npm/express@4.18.2\",\"version\":\"4.18.2\"},{\"purl\":\"pkg:npm/body-parser@1.20.1\",\"version\":\"1.20.1\"},{\"purl\":\"pkg:npm/cookie@0.5.0\",\"version\":\"0.5.0\"}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Transitive dependencies successfully retrieved\"}}2\xe9\x03\n\x0c\x44\x65pendencies\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/v2/dependencies/echo:\x01*\x12\x9f\x01\n\x0fGetDependencies\x12..scanoss.api.dependencies.v2.DependencyRequest\x1a/.scanoss.api.dependencies.v2.DependencyResponse\"+\x88\x02\x01\x82\xd3\xe4\x93\x02\"\"\x1d/v2/dependencies/dependencies:\x01*\x12\xc3\x01\n\x19GetTransitiveDependencies\x12\x38.scanoss.api.dependencies.v2.TransitiveDependencyRequest\x1a\x39.scanoss.api.dependencies.v2.TransitiveDependencyResponse\"1\x82\xd3\xe4\x93\x02+\"&/v2/dependencies/transitive/components:\x01*B\x9c\x02Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\x92\x41\xdd\x01\x12w\n\x1aSCANOSS Dependency Service\"T\n\x14scanoss-dependencies\x12\'https://github.com/scanoss/dependencies\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -35,32 +35,40 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'Z9github.amrom.workers.dev/scanoss/papi/api/dependenciesv2;dependenciesv2\222A\335\001\022w\n\032SCANOSS Dependency Service\"T\n\024scanoss-dependencies\022\'https://github.com/scanoss/dependencies\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_DEPENDENCYREQUEST']._loaded_options = None + _globals['_DEPENDENCYREQUEST']._serialized_options = b'\030\001\222Ad\nbJ`{\"files\":[{\"file\":\"package.json\",\"purls\":[{\"purl\":\"pkg:npm/express\",\"requirement\":\"^4.18.0\"}]}]}' + _globals['_DEPENDENCYRESPONSE']._loaded_options = None + _globals['_DEPENDENCYRESPONSE']._serialized_options = b'\030\001\222A\234\003\n\231\003J\226\003{\"files\":[{\"file\":\"package.json\",\"id\":\"dependency\",\"status\":\"pending\",\"dependencies\":[{\"component\":\"express\",\"purl\":\"pkg:npm/express\",\"version\":\"4.18.2\",\"licenses\":[{\"name\":\"MIT\",\"spdx_id\":\"MIT\",\"is_spdx_approved\":true,\"url\":\"https://opensource.org/licenses/MIT\"}],\"url\":\"https://www.npmjs.com/package/express\",\"comment\":\"\"}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Dependencies successfully retrieved\"}}' + _globals['_TRANSITIVEDEPENDENCYREQUEST']._loaded_options = None + _globals['_TRANSITIVEDEPENDENCYREQUEST']._serialized_options = b'\222A\216\001\n\213\001J\210\001{\"depth\":3,\"limit\":50,\"components\":[{\"purl\":\"pkg:npm/express\",\"requirement\":\"4.18.0\"},{\"purl\":\"pkg:npm/lodash\",\"requirement\":\"4.17.0\"}]}' + _globals['_TRANSITIVEDEPENDENCYRESPONSE']._loaded_options = None + _globals['_TRANSITIVEDEPENDENCYRESPONSE']._serialized_options = b'\222A\221\002\n\216\002J\213\002{\"dependencies\":[{\"purl\":\"pkg:npm/express@4.18.2\",\"version\":\"4.18.2\"},{\"purl\":\"pkg:npm/body-parser@1.20.1\",\"version\":\"1.20.1\"},{\"purl\":\"pkg:npm/cookie@0.5.0\",\"version\":\"0.5.0\"}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Transitive dependencies successfully retrieved\"}}' _globals['_DEPENDENCIES'].methods_by_name['Echo']._loaded_options = None - _globals['_DEPENDENCIES'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/dependencies/echo:\001*' + _globals['_DEPENDENCIES'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/v2/dependencies/echo:\001*' _globals['_DEPENDENCIES'].methods_by_name['GetDependencies']._loaded_options = None - _globals['_DEPENDENCIES'].methods_by_name['GetDependencies']._serialized_options = b'\202\323\344\223\002&\"!/api/v2/dependencies/dependencies:\001*' + _globals['_DEPENDENCIES'].methods_by_name['GetDependencies']._serialized_options = b'\210\002\001\202\323\344\223\002\"\"\035/v2/dependencies/dependencies:\001*' _globals['_DEPENDENCIES'].methods_by_name['GetTransitiveDependencies']._loaded_options = None - _globals['_DEPENDENCIES'].methods_by_name['GetTransitiveDependencies']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/dependencies/transitive:\001*' + _globals['_DEPENDENCIES'].methods_by_name['GetTransitiveDependencies']._serialized_options = b'\202\323\344\223\002+\"&/v2/dependencies/transitive/components:\001*' _globals['_DEPENDENCYREQUEST']._serialized_start=210 - _globals['_DEPENDENCYREQUEST']._serialized_end=449 + _globals['_DEPENDENCYREQUEST']._serialized_end=556 _globals['_DEPENDENCYREQUEST_PURLS']._serialized_start=315 _globals['_DEPENDENCYREQUEST_PURLS']._serialized_end=357 _globals['_DEPENDENCYREQUEST_FILES']._serialized_start=359 _globals['_DEPENDENCYREQUEST_FILES']._serialized_end=449 - _globals['_DEPENDENCYRESPONSE']._serialized_start=452 - _globals['_DEPENDENCYRESPONSE']._serialized_end=988 - _globals['_DEPENDENCYRESPONSE_LICENSES']._serialized_start=599 - _globals['_DEPENDENCYRESPONSE_LICENSES']._serialized_end=679 - _globals['_DEPENDENCYRESPONSE_DEPENDENCIES']._serialized_start=682 - _globals['_DEPENDENCYRESPONSE_DEPENDENCIES']._serialized_end=852 - _globals['_DEPENDENCYRESPONSE_FILES']._serialized_start=855 - _globals['_DEPENDENCYRESPONSE_FILES']._serialized_end=988 - _globals['_TRANSITIVEDEPENDENCYREQUEST']._serialized_start=990 - _globals['_TRANSITIVEDEPENDENCYREQUEST']._serialized_end=1112 - _globals['_TRANSITIVEDEPENDENCYRESPONSE']._serialized_start=1115 - _globals['_TRANSITIVEDEPENDENCYRESPONSE']._serialized_end=1341 - _globals['_TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES']._serialized_start=1296 - _globals['_TRANSITIVEDEPENDENCYRESPONSE_DEPENDENCIES']._serialized_end=1341 - _globals['_DEPENDENCIES']._serialized_start=1344 - _globals['_DEPENDENCIES']._serialized_end=1831 + _globals['_DEPENDENCYRESPONSE']._serialized_start=559 + _globals['_DEPENDENCYRESPONSE']._serialized_end=1543 + _globals['_DEPENDENCYRESPONSE_LICENSES']._serialized_start=706 + _globals['_DEPENDENCYRESPONSE_LICENSES']._serialized_end=813 + _globals['_DEPENDENCYRESPONSE_DEPENDENCIES']._serialized_start=816 + _globals['_DEPENDENCYRESPONSE_DEPENDENCIES']._serialized_end=986 + _globals['_DEPENDENCYRESPONSE_FILES']._serialized_start=989 + _globals['_DEPENDENCYRESPONSE_FILES']._serialized_end=1122 + _globals['_TRANSITIVEDEPENDENCYREQUEST']._serialized_start=1546 + _globals['_TRANSITIVEDEPENDENCYREQUEST']._serialized_end=1815 + _globals['_TRANSITIVEDEPENDENCYRESPONSE']._serialized_start=1818 + _globals['_TRANSITIVEDEPENDENCYRESPONSE']._serialized_end=2339 + _globals['_TRANSITIVEDEPENDENCYRESPONSE_COMPONENT']._serialized_start=1996 + _globals['_TRANSITIVEDEPENDENCYRESPONSE_COMPONENT']._serialized_end=2059 + _globals['_DEPENDENCIES']._serialized_start=2342 + _globals['_DEPENDENCIES']._serialized_end=2831 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py index aa4e5b7a..7e8f6ab3 100644 --- a/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py +++ b/src/scanoss/api/dependencies/v2/scanoss_dependencies_pb2_grpc.py @@ -68,6 +68,7 @@ def Echo(self, request, context): def GetDependencies(self, request, context): """Get dependency details + Deprecated: Use /v2/licenses/components instead """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py index e79f7ec6..148ba0fc 100644 --- a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto\x12\x1cscanoss.api.geoprovenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xd1\x03\n\x13\x43ontributorResponse\x12\x46\n\x05purls\x18\x01 \x03(\x0b\x32\x37.scanoss.api.geoprovenance.v2.ContributorResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x32\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x1a\x31\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\x1a\xd3\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12^\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32\x42.scanoss.api.geoprovenance.v2.ContributorResponse.DeclaredLocation\x12\\\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32\x41.scanoss.api.geoprovenance.v2.ContributorResponse.CuratedLocation\"\x99\x02\n\x0eOriginResponse\x12\x41\n\x05purls\x18\x01 \x03(\x0b\x32\x32.scanoss.api.geoprovenance.v2.OriginResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a,\n\x08Location\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\npercentage\x18\x02 \x01(\x02\x1a_\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12H\n\tlocations\x18\x02 \x03(\x0b\x32\x35.scanoss.api.geoprovenance.v2.OriginResponse.Location2\xb9\x03\n\rGeoProvenance\x12v\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"%\x82\xd3\xe4\x93\x02\x1f\"\x1a/api/v2/geoprovenance/echo:\x01*\x12\x9d\x01\n\x18GetComponentContributors\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.geoprovenance.v2.ContributorResponse\"*\x82\xd3\xe4\x93\x02$\"\x1f/api/v2/geoprovenance/countries:\x01*\x12\x8f\x01\n\x12GetComponentOrigin\x12\".scanoss.api.common.v2.PurlRequest\x1a,.scanoss.api.geoprovenance.v2.OriginResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/geoprovenance/origin:\x01*B\xa4\x02Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\x92\x41\xe3\x01\x12}\n\x1eSCANOSS GEO Provenance Service\"V\n\x15scanoss-geoprovenance\x12(https://github.com/scanoss/geoprovenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n8scanoss/api/geoprovenance/v2/scanoss-geoprovenance.proto\x12\x1cscanoss.api.geoprovenance.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"2\n\x10\x44\x65\x63laredLocation\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\"1\n\x0f\x43uratedLocation\x12\x0f\n\x07\x63ountry\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x05\"\xed\x02\n\x13\x43ontributorResponse\x12\x46\n\x05purls\x18\x01 \x03(\x0b\x32\x37.scanoss.api.geoprovenance.v2.ContributorResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\xd2\x01\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12^\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32..scanoss.api.geoprovenance.v2.DeclaredLocationR\x12\x64\x65\x63lared_locations\x12[\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32-.scanoss.api.geoprovenance.v2.CuratedLocationR\x11\x63urated_locations:\x02\x18\x01\"\xe2\x01\n\x15\x43omponentLocationInfo\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12^\n\x12\x64\x65\x63lared_locations\x18\x02 \x03(\x0b\x32..scanoss.api.geoprovenance.v2.DeclaredLocationR\x12\x64\x65\x63lared_locations\x12[\n\x11\x63urated_locations\x18\x03 \x03(\x0b\x32-.scanoss.api.geoprovenance.v2.CuratedLocationR\x11\x63urated_locations\"\xd5\x04\n\x1d\x43omponentsContributorResponse\x12g\n\x14\x63omponents_locations\x18\x01 \x03(\x0b\x32\x33.scanoss.api.geoprovenance.v2.ComponentLocationInfoR\x14\x63omponents_locations\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x93\x03\x92\x41\x8f\x03\n\x8c\x03J\x89\x03{\"components_locations\":[{\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"declared_locations\":[{\"type\":\"owner\",\"location\":\"Barcelona, Spain\"},{\"type\":\"contributor\",\"location\":\"Berlin, Germany\"}],\"curated_locations\":[{\"country\":\"Spain\",\"count\":8},{\"country\":\"Germany\",\"count\":3},{\"country\":\"United States\",\"count\":2}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance successfully retrieved\"}}\"\xce\x04\n\x1c\x43omponentContributorResponse\x12\x65\n\x13\x63omponent_locations\x18\x01 \x01(\x0b\x32\x33.scanoss.api.geoprovenance.v2.ComponentLocationInfoR\x13\x63omponent_locations\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x8f\x03\x92\x41\x8b\x03\n\x88\x03J\x85\x03{\"component_location\":{\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"declared_locations\":[{\"type\":\"owner\",\"location\":\"Barcelona, Spain\"},{\"type\":\"contributor\",\"location\":\"Berlin, Germany\"}],\"curated_locations\":[{\"country\":\"Spain\",\"count\":8},{\"country\":\"Germany\",\"count\":3},{\"country\":\"United States\",\"count\":2}]},\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance successfully retrieved\"}}\",\n\x08Location\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\npercentage\x18\x02 \x01(\x02\"\\\n\x11\x43omponentLocation\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x39\n\tlocations\x18\x02 \x03(\x0b\x32&.scanoss.api.geoprovenance.v2.Location\"\xe0\x01\n\x0eOriginResponse\x12\x41\n\x05purls\x18\x01 \x03(\x0b\x32\x32.scanoss.api.geoprovenance.v2.OriginResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aP\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x39\n\tlocations\x18\x02 \x03(\x0b\x32&.scanoss.api.geoprovenance.v2.Location:\x02\x18\x01\"\xcd\x03\n\x18\x43omponentsOriginResponse\x12\x63\n\x14\x63omponents_locations\x18\x01 \x03(\x0b\x32/.scanoss.api.geoprovenance.v2.ComponentLocationR\x14\x63omponents_locations\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x94\x02\x92\x41\x90\x02\n\x8d\x02J\x8a\x02{\"components_locations\":[{\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"locations\":[{\"name\":\"ES\",\"percentage\":65.5},{\"name\":\"DE\",\"percentage\":20.3},{\"name\":\"US\",\"percentage\":14.2}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance origin successfully retrieved\"}}\"\xc8\x03\n\x17\x43omponentOriginResponse\x12\x61\n\x13\x63omponent_locations\x18\x01 \x01(\x0b\x32/.scanoss.api.geoprovenance.v2.ComponentLocationR\x13\x63omponent_locations\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x92\x02\x92\x41\x8e\x02\n\x8b\x02J\x88\x02{\"component_locations\": {\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"locations\":[{\"name\":\"ES\",\"percentage\":65.5},{\"name\":\"DE\",\"percentage\":20.3},{\"name\":\"US\",\"percentage\":14.2}]},\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance origin successfully retrieved\"}}2\xff\x08\n\rGeoProvenance\x12r\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"!\x82\xd3\xe4\x93\x02\x1b\"\x16/v2/geoprovenance/echo:\x01*\x12\x9c\x01\n\x18GetComponentContributors\x12\".scanoss.api.common.v2.PurlRequest\x1a\x31.scanoss.api.geoprovenance.v2.ContributorResponse\")\x88\x02\x01\x82\xd3\xe4\x93\x02 \"\x1b/v2/geoprovenance/countries:\x01*\x12\xbe\x01\n\"GetCountryContributorsByComponents\x12(.scanoss.api.common.v2.ComponentsRequest\x1a;.scanoss.api.geoprovenance.v2.ComponentsContributorResponse\"1\x82\xd3\xe4\x93\x02+\"&/v2/geoprovenance/countries/components:\x01*\x12\xb7\x01\n!GetCountryContributorsByComponent\x12\'.scanoss.api.common.v2.ComponentRequest\x1a:.scanoss.api.geoprovenance.v2.ComponentContributorResponse\"-\x82\xd3\xe4\x93\x02\'\x12%/v2/geoprovenance/countries/component\x12\x8e\x01\n\x12GetComponentOrigin\x12\".scanoss.api.common.v2.PurlRequest\x1a,.scanoss.api.geoprovenance.v2.OriginResponse\"&\x88\x02\x01\x82\xd3\xe4\x93\x02\x1d\"\x18/v2/geoprovenance/origin:\x01*\x12\xa9\x01\n\x15GetOriginByComponents\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x36.scanoss.api.geoprovenance.v2.ComponentsOriginResponse\".\x82\xd3\xe4\x93\x02(\"#/v2/geoprovenance/origin/components:\x01*\x12\xa2\x01\n\x14GetOriginByComponent\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x35.scanoss.api.geoprovenance.v2.ComponentOriginResponse\"*\x82\xd3\xe4\x93\x02$\x12\"/v2/geoprovenance/origin/componentB\xa4\x02Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\x92\x41\xe3\x01\x12}\n\x1eSCANOSS GEO Provenance Service\"V\n\x15scanoss-geoprovenance\x12(https://github.com/scanoss/geoprovenance\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -35,26 +35,58 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'Z;github.com/scanoss/papi/api/geoprovenancev2;geoprovenancev2\222A\343\001\022}\n\036SCANOSS GEO Provenance Service\"V\n\025scanoss-geoprovenance\022(https://github.com/scanoss/geoprovenance\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_CONTRIBUTORRESPONSE']._loaded_options = None + _globals['_CONTRIBUTORRESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTSCONTRIBUTORRESPONSE']._loaded_options = None + _globals['_COMPONENTSCONTRIBUTORRESPONSE']._serialized_options = b'\222A\217\003\n\214\003J\211\003{\"components_locations\":[{\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"declared_locations\":[{\"type\":\"owner\",\"location\":\"Barcelona, Spain\"},{\"type\":\"contributor\",\"location\":\"Berlin, Germany\"}],\"curated_locations\":[{\"country\":\"Spain\",\"count\":8},{\"country\":\"Germany\",\"count\":3},{\"country\":\"United States\",\"count\":2}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance successfully retrieved\"}}' + _globals['_COMPONENTCONTRIBUTORRESPONSE']._loaded_options = None + _globals['_COMPONENTCONTRIBUTORRESPONSE']._serialized_options = b'\222A\213\003\n\210\003J\205\003{\"component_location\":{\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"declared_locations\":[{\"type\":\"owner\",\"location\":\"Barcelona, Spain\"},{\"type\":\"contributor\",\"location\":\"Berlin, Germany\"}],\"curated_locations\":[{\"country\":\"Spain\",\"count\":8},{\"country\":\"Germany\",\"count\":3},{\"country\":\"United States\",\"count\":2}]},\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance successfully retrieved\"}}' + _globals['_ORIGINRESPONSE']._loaded_options = None + _globals['_ORIGINRESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTSORIGINRESPONSE']._loaded_options = None + _globals['_COMPONENTSORIGINRESPONSE']._serialized_options = b'\222A\220\002\n\215\002J\212\002{\"components_locations\":[{\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"locations\":[{\"name\":\"ES\",\"percentage\":65.5},{\"name\":\"DE\",\"percentage\":20.3},{\"name\":\"US\",\"percentage\":14.2}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance origin successfully retrieved\"}}' + _globals['_COMPONENTORIGINRESPONSE']._loaded_options = None + _globals['_COMPONENTORIGINRESPONSE']._serialized_options = b'\222A\216\002\n\213\002J\210\002{\"component_locations\": {\"purl\":\"pkg:github/scanoss/engine@5.0.0\",\"locations\":[{\"name\":\"ES\",\"percentage\":65.5},{\"name\":\"DE\",\"percentage\":20.3},{\"name\":\"US\",\"percentage\":14.2}]},\"status\":{\"status\":\"SUCCESS\",\"message\":\"Geo-provenance origin successfully retrieved\"}}' _globals['_GEOPROVENANCE'].methods_by_name['Echo']._loaded_options = None - _globals['_GEOPROVENANCE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\037\"\032/api/v2/geoprovenance/echo:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\033\"\026/v2/geoprovenance/echo:\001*' _globals['_GEOPROVENANCE'].methods_by_name['GetComponentContributors']._loaded_options = None - _globals['_GEOPROVENANCE'].methods_by_name['GetComponentContributors']._serialized_options = b'\202\323\344\223\002$\"\037/api/v2/geoprovenance/countries:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['GetComponentContributors']._serialized_options = b'\210\002\001\202\323\344\223\002 \"\033/v2/geoprovenance/countries:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['GetCountryContributorsByComponents']._loaded_options = None + _globals['_GEOPROVENANCE'].methods_by_name['GetCountryContributorsByComponents']._serialized_options = b'\202\323\344\223\002+\"&/v2/geoprovenance/countries/components:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['GetCountryContributorsByComponent']._loaded_options = None + _globals['_GEOPROVENANCE'].methods_by_name['GetCountryContributorsByComponent']._serialized_options = b'\202\323\344\223\002\'\022%/v2/geoprovenance/countries/component' _globals['_GEOPROVENANCE'].methods_by_name['GetComponentOrigin']._loaded_options = None - _globals['_GEOPROVENANCE'].methods_by_name['GetComponentOrigin']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/geoprovenance/origin:\001*' - _globals['_CONTRIBUTORRESPONSE']._serialized_start=213 - _globals['_CONTRIBUTORRESPONSE']._serialized_end=678 - _globals['_CONTRIBUTORRESPONSE_DECLAREDLOCATION']._serialized_start=363 - _globals['_CONTRIBUTORRESPONSE_DECLAREDLOCATION']._serialized_end=413 - _globals['_CONTRIBUTORRESPONSE_CURATEDLOCATION']._serialized_start=415 - _globals['_CONTRIBUTORRESPONSE_CURATEDLOCATION']._serialized_end=464 + _globals['_GEOPROVENANCE'].methods_by_name['GetComponentOrigin']._serialized_options = b'\210\002\001\202\323\344\223\002\035\"\030/v2/geoprovenance/origin:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['GetOriginByComponents']._loaded_options = None + _globals['_GEOPROVENANCE'].methods_by_name['GetOriginByComponents']._serialized_options = b'\202\323\344\223\002(\"#/v2/geoprovenance/origin/components:\001*' + _globals['_GEOPROVENANCE'].methods_by_name['GetOriginByComponent']._loaded_options = None + _globals['_GEOPROVENANCE'].methods_by_name['GetOriginByComponent']._serialized_options = b'\202\323\344\223\002$\022\"/v2/geoprovenance/origin/component' + _globals['_DECLAREDLOCATION']._serialized_start=212 + _globals['_DECLAREDLOCATION']._serialized_end=262 + _globals['_CURATEDLOCATION']._serialized_start=264 + _globals['_CURATEDLOCATION']._serialized_end=313 + _globals['_CONTRIBUTORRESPONSE']._serialized_start=316 + _globals['_CONTRIBUTORRESPONSE']._serialized_end=681 _globals['_CONTRIBUTORRESPONSE_PURLS']._serialized_start=467 - _globals['_CONTRIBUTORRESPONSE_PURLS']._serialized_end=678 - _globals['_ORIGINRESPONSE']._serialized_start=681 - _globals['_ORIGINRESPONSE']._serialized_end=962 - _globals['_ORIGINRESPONSE_LOCATION']._serialized_start=821 - _globals['_ORIGINRESPONSE_LOCATION']._serialized_end=865 - _globals['_ORIGINRESPONSE_PURLS']._serialized_start=867 - _globals['_ORIGINRESPONSE_PURLS']._serialized_end=962 - _globals['_GEOPROVENANCE']._serialized_start=965 - _globals['_GEOPROVENANCE']._serialized_end=1406 + _globals['_CONTRIBUTORRESPONSE_PURLS']._serialized_end=677 + _globals['_COMPONENTLOCATIONINFO']._serialized_start=684 + _globals['_COMPONENTLOCATIONINFO']._serialized_end=910 + _globals['_COMPONENTSCONTRIBUTORRESPONSE']._serialized_start=913 + _globals['_COMPONENTSCONTRIBUTORRESPONSE']._serialized_end=1510 + _globals['_COMPONENTCONTRIBUTORRESPONSE']._serialized_start=1513 + _globals['_COMPONENTCONTRIBUTORRESPONSE']._serialized_end=2103 + _globals['_LOCATION']._serialized_start=2105 + _globals['_LOCATION']._serialized_end=2149 + _globals['_COMPONENTLOCATION']._serialized_start=2151 + _globals['_COMPONENTLOCATION']._serialized_end=2243 + _globals['_ORIGINRESPONSE']._serialized_start=2246 + _globals['_ORIGINRESPONSE']._serialized_end=2470 + _globals['_ORIGINRESPONSE_PURLS']._serialized_start=2386 + _globals['_ORIGINRESPONSE_PURLS']._serialized_end=2466 + _globals['_COMPONENTSORIGINRESPONSE']._serialized_start=2473 + _globals['_COMPONENTSORIGINRESPONSE']._serialized_end=2934 + _globals['_COMPONENTORIGINRESPONSE']._serialized_start=2937 + _globals['_COMPONENTORIGINRESPONSE']._serialized_end=3393 + _globals['_GEOPROVENANCE']._serialized_start=3396 + _globals['_GEOPROVENANCE']._serialized_end=4547 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py index d669a5e4..5ab07c9a 100644 --- a/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py +++ b/src/scanoss/api/geoprovenance/v2/scanoss_geoprovenance_pb2_grpc.py @@ -47,11 +47,31 @@ def __init__(self, channel): request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.FromString, _registered_method=True) + self.GetCountryContributorsByComponents = channel.unary_unary( + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetCountryContributorsByComponents', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentsContributorResponse.FromString, + _registered_method=True) + self.GetCountryContributorsByComponent = channel.unary_unary( + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetCountryContributorsByComponent', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentContributorResponse.FromString, + _registered_method=True) self.GetComponentOrigin = channel.unary_unary( '/scanoss.api.geoprovenance.v2.GeoProvenance/GetComponentOrigin', request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.FromString, _registered_method=True) + self.GetOriginByComponents = channel.unary_unary( + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetOriginByComponents', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentsOriginResponse.FromString, + _registered_method=True) + self.GetOriginByComponent = channel.unary_unary( + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetOriginByComponent', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentOriginResponse.FromString, + _registered_method=True) class GeoProvenanceServicer(object): @@ -60,21 +80,61 @@ class GeoProvenanceServicer(object): """ def Echo(self, request, context): - """Standard echo + """Standard health check endpoint to verify service availability and connectivity """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetComponentContributors(self, request, context): + """[DEPRECATED] Get component-level Geo Provenance based on contributor declared location + This method accepts PURL-based requests and is deprecated in favor of GetCountryContributorsByComponent + which accepts ComponentRequest for better component identification + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetCountryContributorsByComponents(self, request, context): + """Get component-level Geo Provenance based on contributor declared location + This is the current method that accepts ComponentsRequest for enhanced component identification + Replaces the deprecated GetComponentContributors method + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetCountryContributorsByComponent(self, request, context): """Get component-level Geo Provenance based on contributor declared location + This is the current method that accepts ComponentRequest for enhanced component identification + Replaces the deprecated GetComponentContributors method """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetComponentOrigin(self, request, context): + """[DEPRECATED] Get component-level Geo Provenance based on contributor origin commit times + This method accepts PURL-based requests and is deprecated in favor of GetOriginByComponent + which accepts ComponentRequest for better component identification + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetOriginByComponents(self, request, context): """Get component-level Geo Provenance based on contributor origin commit times + This is the current method that accepts ComponentsRequest for enhanced component identification + Replaces the deprecated GetComponentOrigin method + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetOriginByComponent(self, request, context): + """Get component-level Geo Provenance based on contributor origin commit times + This is the current method that accepts ComponentRequest for enhanced component identification + Replaces the deprecated GetComponentOrigin method """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -93,11 +153,31 @@ def add_GeoProvenanceServicer_to_server(servicer, server): request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ContributorResponse.SerializeToString, ), + 'GetCountryContributorsByComponents': grpc.unary_unary_rpc_method_handler( + servicer.GetCountryContributorsByComponents, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentsContributorResponse.SerializeToString, + ), + 'GetCountryContributorsByComponent': grpc.unary_unary_rpc_method_handler( + servicer.GetCountryContributorsByComponent, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentContributorResponse.SerializeToString, + ), 'GetComponentOrigin': grpc.unary_unary_rpc_method_handler( servicer.GetComponentOrigin, request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.OriginResponse.SerializeToString, ), + 'GetOriginByComponents': grpc.unary_unary_rpc_method_handler( + servicer.GetOriginByComponents, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentsOriginResponse.SerializeToString, + ), + 'GetOriginByComponent': grpc.unary_unary_rpc_method_handler( + servicer.GetOriginByComponent, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentOriginResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.geoprovenance.v2.GeoProvenance', rpc_method_handlers) @@ -165,6 +245,60 @@ def GetComponentContributors(request, metadata, _registered_method=True) + @staticmethod + def GetCountryContributorsByComponents(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetCountryContributorsByComponents', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentsContributorResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetCountryContributorsByComponent(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetCountryContributorsByComponent', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentContributorResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def GetComponentOrigin(request, target, @@ -191,3 +325,57 @@ def GetComponentOrigin(request, timeout, metadata, _registered_method=True) + + @staticmethod + def GetOriginByComponents(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetOriginByComponents', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentsOriginResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetOriginByComponent(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.geoprovenance.v2.GeoProvenance/GetOriginByComponent', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_geoprovenance_dot_v2_dot_scanoss__geoprovenance__pb2.ComponentOriginResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py b/src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py index 144f8d7d..1476d0e9 100644 --- a/src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py +++ b/src/scanoss/api/licenses/v2/scanoss_licenses_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/licenses/v2/scanoss-licenses.proto\x12\x17scanoss.api.licenses.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xbd\x03\n\x18\x43omponentLicenseResponse\x12@\n\tcomponent\x18\x01 \x01(\x0b\x32-.scanoss.api.licenses.v2.ComponentLicenseInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xa7\x02\x92\x41\xa3\x02\n\xa0\x02J\x9d\x02{\"component\":{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}\"\xe9\x04\n\x19\x43omponentsLicenseResponse\x12\x41\n\ncomponents\x18\x01 \x03(\x0b\x32-.scanoss.api.licenses.v2.ComponentLicenseInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xd1\x03\x92\x41\xcd\x03\n\xca\x03J\xc7\x03{\"components\":[{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py@v1.30.0\",\"requirement\": \"\",\"version\": \"v1.30.0\",\"statement\": \"MIT\", \"licenses\": [{\"id\": \"MIT\",\"full_name\": \"MIT License\"}]} ], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}\"\x89\x01\n\x16LicenseDetailsResponse\x12\x38\n\x07license\x18\x01 \x01(\x0b\x32\'.scanoss.api.licenses.v2.LicenseDetails\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\"\x81\x01\n\x13ObligationsResponse\x12\x33\n\x0bobligations\x18\x01 \x01(\x0b\x32\x1e.scanoss.api.licenses.v2.OSADL\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\"\xa3\x04\n\x04SPDX\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tfull_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65tails_url\x18\x04 \x01(\t\x12\x15\n\rreference_url\x18\x05 \x01(\t\x12\x15\n\ris_deprecated\x18\x06 \x01(\x08\x12\x14\n\x0cis_fsf_libre\x18\x07 \x01(\x08\x12\x17\n\x0fis_osi_approved\x18\x08 \x01(\x08\x12\x10\n\x08see_also\x18\t \x03(\t\x12>\n\ncross_refs\x18\n \x03(\x0b\x32*.scanoss.api.licenses.v2.SPDX.SPDXCrossRef\x12?\n\nexceptions\x18\x0b \x03(\x0b\x32+.scanoss.api.licenses.v2.SPDX.SPDXException\x1a\x88\x01\n\x0cSPDXCrossRef\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x10\n\x08is_valid\x18\x02 \x01(\x08\x12\x0f\n\x07is_live\x18\x03 \x01(\x08\x12\x11\n\ttimestamp\x18\x04 \x01(\t\x12\x17\n\x0fis_wayback_link\x18\x05 \x01(\x08\x12\r\n\x05order\x18\x06 \x01(\x05\x12\r\n\x05match\x18\x07 \x01(\t\x1al\n\rSPDXException\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tfull_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65tails_url\x18\x03 \x01(\t\x12\x10\n\x08see_also\x18\x05 \x03(\t\x12\x15\n\ris_deprecated\x18\x06 \x01(\x08\"\x97\x02\n\x05OSADL\x12\x17\n\x0f\x63opyleft_clause\x18\x01 \x01(\x08\x12\x14\n\x0cpatent_hints\x18\x02 \x01(\x08\x12\x15\n\rcompatibility\x18\x03 \x03(\t\x12\x1f\n\x17\x64\x65pending_compatibility\x18\x04 \x03(\t\x12\x17\n\x0fincompatibility\x18\x05 \x03(\t\x12>\n\tuse_cases\x18\x06 \x03(\x0b\x32+.scanoss.api.licenses.v2.OSADL.OSADLUseCase\x1aN\n\x0cOSADLUseCase\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0fobligation_text\x18\x02 \x01(\t\x12\x17\n\x0fobligation_json\x18\x03 \x01(\t\"{\n\x0bLicenseInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tfull_name\x18\x02 \x01(\t:M\x92\x41J\nHJF{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}\"\xb3\x01\n\x0eLicenseDetails\x12\x11\n\tfull_name\x18\x01 \x01(\t\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32$.scanoss.api.licenses.v2.LicenseType\x12+\n\x04spdx\x18\x03 \x01(\x0b\x32\x1d.scanoss.api.licenses.v2.SPDX\x12-\n\x05osadl\x18\x04 \x01(\x0b\x32\x1e.scanoss.api.licenses.v2.OSADL\"\x1c\n\x0eLicenseRequest\x12\n\n\x02id\x18\x01 \x01(\t\"\x95\x01\n\x14\x43omponentLicenseInfo\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x11\n\tstatement\x18\x04 \x01(\t\x12\x36\n\x08licenses\x18\x05 \x03(\x0b\x32$.scanoss.api.licenses.v2.LicenseInfo*l\n\x0bLicenseType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0e\n\nPERMISSIVE\x10\x01\x12\x0c\n\x08\x43OPYLEFT\x10\x02\x12\x0e\n\nCOMMERCIAL\x10\x03\x12\x0f\n\x0bPROPRIETARY\x10\x04\x12\x11\n\rPUBLIC_DOMAIN\x10\x05\x32\xd0\x05\n\x07License\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/licenses/echo:\x01*\x12\x96\x01\n\x14GetComponentLicenses\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x31.scanoss.api.licenses.v2.ComponentLicenseResponse\"\"\x82\xd3\xe4\x93\x02\x1c\x12\x1a/api/v2/licenses/component\x12\x9d\x01\n\x15GetComponentsLicenses\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x32.scanoss.api.licenses.v2.ComponentsLicenseResponse\"&\x82\xd3\xe4\x93\x02 \"\x1b/api/v2/licenses/components:\x01*\x12\x88\x01\n\nGetDetails\x12\'.scanoss.api.licenses.v2.LicenseRequest\x1a/.scanoss.api.licenses.v2.LicenseDetailsResponse\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/api/v2/licenses/details\x12\x8d\x01\n\x0eGetObligations\x12\'.scanoss.api.licenses.v2.LicenseRequest\x1a,.scanoss.api.licenses.v2.ObligationsResponse\"$\x82\xd3\xe4\x93\x02\x1e\x12\x1c/api/v2/licenses/obligationsB\xa7\x02Z1github.amrom.workers.dev/scanoss/papi/api/licensesv2;licensesv2\x92\x41\xf0\x01\x12\xb4\x01\n\x17SCANOSS License Service\x12\x46License service provides license intelligence for software components.\"L\n\x10scanoss-licenses\x12#https://github.com/scanoss/licenses\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x01\x02\x32\x10\x61pplication/json:\x10\x61pplication/jsonb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/licenses/v2/scanoss-licenses.proto\x12\x17scanoss.api.licenses.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xbd\x03\n\x18\x43omponentLicenseResponse\x12@\n\tcomponent\x18\x01 \x01(\x0b\x32-.scanoss.api.licenses.v2.ComponentLicenseInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xa7\x02\x92\x41\xa3\x02\n\xa0\x02J\x9d\x02{\"component\":{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}\"\xe9\x04\n\x19\x43omponentsLicenseResponse\x12\x41\n\ncomponents\x18\x01 \x03(\x0b\x32-.scanoss.api.licenses.v2.ComponentLicenseInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xd1\x03\x92\x41\xcd\x03\n\xca\x03J\xc7\x03{\"components\":[{\"purl\": \"pkg:github/scanoss/engine@1.0.0\", \"requirement\": \"\", \"version\": \"1.0.0\", \"statement\": \"GPL-2.0\", \"licenses\": [{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py@v1.30.0\",\"requirement\": \"\",\"version\": \"v1.30.0\",\"statement\": \"MIT\", \"licenses\": [{\"id\": \"MIT\",\"full_name\": \"MIT License\"}]} ], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Licenses Successfully retrieved\"}}\"\x89\x01\n\x16LicenseDetailsResponse\x12\x38\n\x07license\x18\x01 \x01(\x0b\x32\'.scanoss.api.licenses.v2.LicenseDetails\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\"\x81\x01\n\x13ObligationsResponse\x12\x33\n\x0bobligations\x18\x01 \x01(\x0b\x32\x1e.scanoss.api.licenses.v2.OSADL\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\"\xe4\x05\n\x04SPDX\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\tfull_name\x18\x02 \x01(\tR\tfull_name\x12 \n\x0b\x64\x65tails_url\x18\x04 \x01(\tR\x0b\x64\x65tails_url\x12$\n\rreference_url\x18\x05 \x01(\tR\rreference_url\x12$\n\ris_deprecated\x18\x06 \x01(\x08R\ris_deprecated\x12\"\n\x0cis_fsf_libre\x18\x07 \x01(\x08R\x0cis_fsf_libre\x12(\n\x0fis_osi_approved\x18\x08 \x01(\x08R\x0fis_osi_approved\x12\x1a\n\x08see_also\x18\t \x03(\tR\x08see_also\x12J\n\ncross_refs\x18\n \x03(\x0b\x32*.scanoss.api.licenses.v2.SPDX.SPDXCrossRefR\ncross_refs\x12?\n\nexceptions\x18\x0b \x03(\x0b\x32+.scanoss.api.licenses.v2.SPDX.SPDXException\x1a\xac\x01\n\x0cSPDXCrossRef\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x1a\n\x08is_valid\x18\x02 \x01(\x08R\x08is_valid\x12\x18\n\x07is_live\x18\x03 \x01(\x08R\x07is_live\x12\x11\n\ttimestamp\x18\x04 \x01(\t\x12(\n\x0fis_wayback_link\x18\x05 \x01(\x08R\x0fis_wayback_link\x12\r\n\x05order\x18\x06 \x01(\x05\x12\r\n\x05match\x18\x07 \x01(\t\x1a\x9d\x01\n\rSPDXException\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\tfull_name\x18\x02 \x01(\tR\tfull_name\x12 \n\x0b\x64\x65tails_url\x18\x03 \x01(\tR\x0b\x64\x65tails_url\x12\x1a\n\x08see_also\x18\x05 \x03(\tR\x08see_also\x12$\n\ris_deprecated\x18\x06 \x01(\x08R\ris_deprecated\"\xfc\x02\n\x05OSADL\x12(\n\x0f\x63opyleft_clause\x18\x01 \x01(\x08R\x0f\x63opyleft_clause\x12\"\n\x0cpatent_hints\x18\x02 \x01(\x08R\x0cpatent_hints\x12\x15\n\rcompatibility\x18\x03 \x03(\t\x12\x38\n\x17\x64\x65pending_compatibility\x18\x04 \x03(\tR\x17\x64\x65pending_compatibility\x12\x17\n\x0fincompatibility\x18\x05 \x03(\t\x12I\n\tuse_cases\x18\x06 \x03(\x0b\x32+.scanoss.api.licenses.v2.OSADL.OSADLUseCaseR\tuse_cases\x1ap\n\x0cOSADLUseCase\x12\x0c\n\x04name\x18\x01 \x01(\t\x12(\n\x0fobligation_text\x18\x02 \x01(\tR\x0fobligation_text\x12(\n\x0fobligation_json\x18\x03 \x01(\tR\x0fobligation_json\"\x86\x01\n\x0bLicenseInfo\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\tfull_name\x18\x02 \x01(\tR\tfull_name:M\x92\x41J\nHJF{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}\"\xbe\x01\n\x0eLicenseDetails\x12\x1c\n\tfull_name\x18\x01 \x01(\tR\tfull_name\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32$.scanoss.api.licenses.v2.LicenseType\x12+\n\x04spdx\x18\x03 \x01(\x0b\x32\x1d.scanoss.api.licenses.v2.SPDX\x12-\n\x05osadl\x18\x04 \x01(\x0b\x32\x1e.scanoss.api.licenses.v2.OSADL\"\x1c\n\x0eLicenseRequest\x12\n\n\x02id\x18\x01 \x01(\t\"\x95\x01\n\x14\x43omponentLicenseInfo\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x13\n\x0brequirement\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t\x12\x11\n\tstatement\x18\x04 \x01(\t\x12\x36\n\x08licenses\x18\x05 \x03(\x0b\x32$.scanoss.api.licenses.v2.LicenseInfo*l\n\x0bLicenseType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0e\n\nPERMISSIVE\x10\x01\x12\x0c\n\x08\x43OPYLEFT\x10\x02\x12\x0e\n\nCOMMERCIAL\x10\x03\x12\x0f\n\x0bPROPRIETARY\x10\x04\x12\x11\n\rPUBLIC_DOMAIN\x10\x05\x32\xbc\x05\n\x07License\x12m\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1c\x82\xd3\xe4\x93\x02\x16\"\x11/v2/licenses/echo:\x01*\x12\x92\x01\n\x14GetComponentLicenses\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x31.scanoss.api.licenses.v2.ComponentLicenseResponse\"\x1e\x82\xd3\xe4\x93\x02\x18\x12\x16/v2/licenses/component\x12\x99\x01\n\x15GetComponentsLicenses\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x32.scanoss.api.licenses.v2.ComponentsLicenseResponse\"\"\x82\xd3\xe4\x93\x02\x1c\"\x17/v2/licenses/components:\x01*\x12\x84\x01\n\nGetDetails\x12\'.scanoss.api.licenses.v2.LicenseRequest\x1a/.scanoss.api.licenses.v2.LicenseDetailsResponse\"\x1c\x82\xd3\xe4\x93\x02\x16\x12\x14/v2/licenses/details\x12\x89\x01\n\x0eGetObligations\x12\'.scanoss.api.licenses.v2.LicenseRequest\x1a,.scanoss.api.licenses.v2.ObligationsResponse\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/v2/licenses/obligationsB\xa7\x02Z1github.amrom.workers.dev/scanoss/papi/api/licensesv2;licensesv2\x92\x41\xf0\x01\x12\xb4\x01\n\x17SCANOSS License Service\x12\x46License service provides license intelligence for software components.\"L\n\x10scanoss-licenses\x12#https://github.com/scanoss/licenses\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x01\x02\x32\x10\x61pplication/json:\x10\x61pplication/jsonb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -42,17 +42,17 @@ _globals['_LICENSEINFO']._loaded_options = None _globals['_LICENSEINFO']._serialized_options = b'\222AJ\nHJF{\"id\": \"GPL-2.0\", \"full_name\": \"GNU General Public License v2.0 only\"}' _globals['_LICENSE'].methods_by_name['Echo']._loaded_options = None - _globals['_LICENSE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/licenses/echo:\001*' + _globals['_LICENSE'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\026\"\021/v2/licenses/echo:\001*' _globals['_LICENSE'].methods_by_name['GetComponentLicenses']._loaded_options = None - _globals['_LICENSE'].methods_by_name['GetComponentLicenses']._serialized_options = b'\202\323\344\223\002\034\022\032/api/v2/licenses/component' + _globals['_LICENSE'].methods_by_name['GetComponentLicenses']._serialized_options = b'\202\323\344\223\002\030\022\026/v2/licenses/component' _globals['_LICENSE'].methods_by_name['GetComponentsLicenses']._loaded_options = None - _globals['_LICENSE'].methods_by_name['GetComponentsLicenses']._serialized_options = b'\202\323\344\223\002 \"\033/api/v2/licenses/components:\001*' + _globals['_LICENSE'].methods_by_name['GetComponentsLicenses']._serialized_options = b'\202\323\344\223\002\034\"\027/v2/licenses/components:\001*' _globals['_LICENSE'].methods_by_name['GetDetails']._loaded_options = None - _globals['_LICENSE'].methods_by_name['GetDetails']._serialized_options = b'\202\323\344\223\002\032\022\030/api/v2/licenses/details' + _globals['_LICENSE'].methods_by_name['GetDetails']._serialized_options = b'\202\323\344\223\002\026\022\024/v2/licenses/details' _globals['_LICENSE'].methods_by_name['GetObligations']._loaded_options = None - _globals['_LICENSE'].methods_by_name['GetObligations']._serialized_options = b'\202\323\344\223\002\036\022\034/api/v2/licenses/obligations' - _globals['_LICENSETYPE']._serialized_start=2858 - _globals['_LICENSETYPE']._serialized_end=2966 + _globals['_LICENSE'].methods_by_name['GetObligations']._serialized_options = b'\202\323\344\223\002\032\022\030/v2/licenses/obligations' + _globals['_LICENSETYPE']._serialized_start=3175 + _globals['_LICENSETYPE']._serialized_end=3283 _globals['_COMPONENTLICENSERESPONSE']._serialized_start=198 _globals['_COMPONENTLICENSERESPONSE']._serialized_end=643 _globals['_COMPONENTSLICENSERESPONSE']._serialized_start=646 @@ -62,23 +62,23 @@ _globals['_OBLIGATIONSRESPONSE']._serialized_start=1406 _globals['_OBLIGATIONSRESPONSE']._serialized_end=1535 _globals['_SPDX']._serialized_start=1538 - _globals['_SPDX']._serialized_end=2085 - _globals['_SPDX_SPDXCROSSREF']._serialized_start=1839 - _globals['_SPDX_SPDXCROSSREF']._serialized_end=1975 - _globals['_SPDX_SPDXEXCEPTION']._serialized_start=1977 - _globals['_SPDX_SPDXEXCEPTION']._serialized_end=2085 - _globals['_OSADL']._serialized_start=2088 - _globals['_OSADL']._serialized_end=2367 - _globals['_OSADL_OSADLUSECASE']._serialized_start=2289 - _globals['_OSADL_OSADLUSECASE']._serialized_end=2367 - _globals['_LICENSEINFO']._serialized_start=2369 - _globals['_LICENSEINFO']._serialized_end=2492 - _globals['_LICENSEDETAILS']._serialized_start=2495 - _globals['_LICENSEDETAILS']._serialized_end=2674 - _globals['_LICENSEREQUEST']._serialized_start=2676 - _globals['_LICENSEREQUEST']._serialized_end=2704 - _globals['_COMPONENTLICENSEINFO']._serialized_start=2707 - _globals['_COMPONENTLICENSEINFO']._serialized_end=2856 - _globals['_LICENSE']._serialized_start=2969 - _globals['_LICENSE']._serialized_end=3689 + _globals['_SPDX']._serialized_end=2278 + _globals['_SPDX_SPDXCROSSREF']._serialized_start=1946 + _globals['_SPDX_SPDXCROSSREF']._serialized_end=2118 + _globals['_SPDX_SPDXEXCEPTION']._serialized_start=2121 + _globals['_SPDX_SPDXEXCEPTION']._serialized_end=2278 + _globals['_OSADL']._serialized_start=2281 + _globals['_OSADL']._serialized_end=2661 + _globals['_OSADL_OSADLUSECASE']._serialized_start=2549 + _globals['_OSADL_OSADLUSECASE']._serialized_end=2661 + _globals['_LICENSEINFO']._serialized_start=2664 + _globals['_LICENSEINFO']._serialized_end=2798 + _globals['_LICENSEDETAILS']._serialized_start=2801 + _globals['_LICENSEDETAILS']._serialized_end=2991 + _globals['_LICENSEREQUEST']._serialized_start=2993 + _globals['_LICENSEREQUEST']._serialized_end=3021 + _globals['_COMPONENTLICENSEINFO']._serialized_start=3024 + _globals['_COMPONENTLICENSEINFO']._serialized_end=3173 + _globals['_LICENSE']._serialized_start=3286 + _globals['_LICENSE']._serialized_end=3986 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py index 90ae2840..6c61c039 100644 --- a/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py +++ b/src/scanoss/api/scanning/v2/scanoss_scanning_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xc5\x03\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x16\n\x0erank_threshold\x18\x02 \x01(\x05\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x1a\xbb\x02\n\x08\x43hildren\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x16\n\x0esim_hash_names\x18\x02 \x01(\t\x12\x18\n\x10sim_hash_content\x18\x03 \x01(\t\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12\x1a\n\x12sim_hash_dir_names\x18\x05 \x01(\t\x12Y\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntry\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\xa3\x03\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a)\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x1a\x94\x01\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0e\n\x06vendor\x18\x03 \x01(\t\x12>\n\x08versions\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHResponse.Version\x12\x0c\n\x04rank\x18\x05 \x01(\x05\x12\r\n\x05order\x18\x06 \x01(\x05\x1a]\n\x06Result\x12\x0f\n\x07path_id\x18\x01 \x01(\t\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\x81\x02\n\x08Scanning\x12q\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/api/v2/scanning/echo:\x01*\x12\x81\x01\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\"$\x82\xd3\xe4\x93\x02\x1e\"\x19/api/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.scanoss/api/scanning/v2/scanoss-scanning.proto\x12\x17scanoss.api.scanning.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\xde\x04\n\nHFHRequest\x12:\n\x04root\x18\x01 \x01(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12&\n\x0erank_threshold\x18\x02 \x01(\x05R\x0erank_threshold\x12\x10\n\x08\x63\x61tegory\x18\x03 \x01(\t\x12\x13\n\x0bquery_limit\x18\x04 \x01(\x05\x12\x1b\n\x13recursive_threshold\x18\x05 \x01(\x02\x12\x1a\n\x12min_accepted_score\x18\x06 \x01(\x02\x1a\x8b\x03\n\x08\x43hildren\x12\x18\n\x07path_id\x18\x01 \x01(\tR\x07path_id\x12&\n\x0esim_hash_names\x18\x02 \x01(\tR\x0esim_hash_names\x12*\n\x10sim_hash_content\x18\x03 \x01(\tR\x10sim_hash_content\x12>\n\x08\x63hildren\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHRequest.Children\x12.\n\x12sim_hash_dir_names\x18\x05 \x01(\tR\x12sim_hash_dir_names\x12j\n\x0flang_extensions\x18\x06 \x03(\x0b\x32@.scanoss.api.scanning.v2.HFHRequest.Children.LangExtensionsEntryR\x0flang_extensions\x1a\x35\n\x13LangExtensionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\"\xac\x03\n\x0bHFHResponse\x12<\n\x07results\x18\x01 \x03(\x0b\x32+.scanoss.api.scanning.v2.HFHResponse.Result\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a)\n\x07Version\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x1a\x94\x01\n\tComponent\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0e\n\x06vendor\x18\x03 \x01(\t\x12>\n\x08versions\x18\x04 \x03(\x0b\x32,.scanoss.api.scanning.v2.HFHResponse.Version\x12\x0c\n\x04rank\x18\x05 \x01(\x05\x12\r\n\x05order\x18\x06 \x01(\x05\x1a\x66\n\x06Result\x12\x18\n\x07path_id\x18\x01 \x01(\tR\x07path_id\x12\x42\n\ncomponents\x18\x02 \x03(\x0b\x32..scanoss.api.scanning.v2.HFHResponse.Component2\xf8\x01\n\x08Scanning\x12m\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1c\x82\xd3\xe4\x93\x02\x16\"\x11/v2/scanning/echo:\x01*\x12}\n\x0e\x46olderHashScan\x12#.scanoss.api.scanning.v2.HFHRequest\x1a$.scanoss.api.scanning.v2.HFHResponse\" \x82\xd3\xe4\x93\x02\x1a\"\x15/v2/scanning/hfh/scan:\x01*B\x8a\x02Z1github.amrom.workers.dev/scanoss/papi/api/scanningv2;scanningv2\x92\x41\xd3\x01\x12m\n\x18SCANOSS Scanning Service\"L\n\x10scanoss-scanning\x12#https://github.com/scanoss/scanning\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -38,23 +38,23 @@ _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._loaded_options = None _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_options = b'8\001' _globals['_SCANNING'].methods_by_name['Echo']._loaded_options = None - _globals['_SCANNING'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\032\"\025/api/v2/scanning/echo:\001*' + _globals['_SCANNING'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\026\"\021/v2/scanning/echo:\001*' _globals['_SCANNING'].methods_by_name['FolderHashScan']._loaded_options = None - _globals['_SCANNING'].methods_by_name['FolderHashScan']._serialized_options = b'\202\323\344\223\002\036\"\031/api/v2/scanning/hfh/scan:\001*' + _globals['_SCANNING'].methods_by_name['FolderHashScan']._serialized_options = b'\202\323\344\223\002\032\"\025/v2/scanning/hfh/scan:\001*' _globals['_HFHREQUEST']._serialized_start=198 - _globals['_HFHREQUEST']._serialized_end=651 - _globals['_HFHREQUEST_CHILDREN']._serialized_start=336 - _globals['_HFHREQUEST_CHILDREN']._serialized_end=651 - _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=598 - _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=651 - _globals['_HFHRESPONSE']._serialized_start=654 - _globals['_HFHRESPONSE']._serialized_end=1073 - _globals['_HFHRESPONSE_VERSION']._serialized_start=786 - _globals['_HFHRESPONSE_VERSION']._serialized_end=827 - _globals['_HFHRESPONSE_COMPONENT']._serialized_start=830 - _globals['_HFHRESPONSE_COMPONENT']._serialized_end=978 - _globals['_HFHRESPONSE_RESULT']._serialized_start=980 - _globals['_HFHRESPONSE_RESULT']._serialized_end=1073 - _globals['_SCANNING']._serialized_start=1076 - _globals['_SCANNING']._serialized_end=1333 + _globals['_HFHREQUEST']._serialized_end=804 + _globals['_HFHREQUEST_CHILDREN']._serialized_start=409 + _globals['_HFHREQUEST_CHILDREN']._serialized_end=804 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_start=751 + _globals['_HFHREQUEST_CHILDREN_LANGEXTENSIONSENTRY']._serialized_end=804 + _globals['_HFHRESPONSE']._serialized_start=807 + _globals['_HFHRESPONSE']._serialized_end=1235 + _globals['_HFHRESPONSE_VERSION']._serialized_start=939 + _globals['_HFHRESPONSE_VERSION']._serialized_end=980 + _globals['_HFHRESPONSE_COMPONENT']._serialized_start=983 + _globals['_HFHRESPONSE_COMPONENT']._serialized_end=1131 + _globals['_HFHRESPONSE_RESULT']._serialized_start=1133 + _globals['_HFHRESPONSE_RESULT']._serialized_end=1235 + _globals['_SCANNING']._serialized_start=1238 + _globals['_SCANNING']._serialized_end=1486 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py index 195d4386..af4a519d 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"\x96\x03\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1a\x43\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\x1a\x64\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12=\n\x06issues\x18\x03 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Issue\x1a\x63\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12;\n\x05\x66iles\x18\x03 \x03(\x0b\x32,.scanoss.api.semgrep.v2.SemgrepResponse.File2\xf8\x01\n\x07Semgrep\x12p\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1f\x82\xd3\xe4\x93\x02\x19\"\x14/api/v2/semgrep/echo:\x01*\x12{\n\tGetIssues\x12\".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse\"!\x82\xd3\xe4\x93\x02\x1b\"\x16/api/v2/semgrep/issues:\x01*B\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service\"J\n\x0fscanoss-semgrep\x12\"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,scanoss/api/semgrep/v2/scanoss-semgrep.proto\x12\x16scanoss.api.semgrep.v2\x1a*scanoss/api/common/v2/scanoss-common.proto\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\"C\n\x05Issue\x12\x0e\n\x06ruleID\x18\x01 \x01(\t\x12\x0c\n\x04\x66rom\x18\x02 \x01(\t\x12\n\n\x02to\x18\x03 \x01(\t\x12\x10\n\x08severity\x18\x04 \x01(\t\"T\n\x04\x46ile\x12\x0f\n\x07\x66ileMD5\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12-\n\x06issues\x18\x03 \x03(\x0b\x32\x1d.scanoss.api.semgrep.v2.Issue\"\xdf\x01\n\x0fSemgrepResponse\x12<\n\x05purls\x18\x01 \x03(\x0b\x32-.scanoss.api.semgrep.v2.SemgrepResponse.Purls\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse\x1aS\n\x05Purls\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12+\n\x05\x66iles\x18\x03 \x03(\x0b\x32\x1c.scanoss.api.semgrep.v2.File:\x02\x18\x01\"u\n\x12\x43omponentIssueInfo\x12\x0c\n\x04purl\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x13\n\x0brequirement\x18\x03 \x01(\t\x12+\n\x05\x66iles\x18\x04 \x03(\x0b\x32\x1c.scanoss.api.semgrep.v2.File\"\xe6\x06\n\x17\x43omponentsIssueResponse\x12>\n\ncomponents\x18\x01 \x03(\x0b\x32*.scanoss.api.semgrep.v2.ComponentIssueInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xd3\x05\x92\x41\xcf\x05\n\xcc\x05J\xc9\x05{\"components\":[{\"purl\":\"pkg:maven/org.apache.commons/commons-lang3\",\"version\":\"3.12.0\",\"requirement\":\"3.12.0\",\"files\":[{\"fileMD5\":\"a1b2c3d4e5f6\",\"path\":\"src/main/java/org/apache/commons/lang3/StringUtils.java\",\"issues\":[{\"ruleID\":\"java.lang.security.audit.crypto.weak-hash\",\"from\":\"156\",\"to\":\"159\",\"severity\":\"WARNING\"},{\"ruleID\":\"java.lang.security.audit.sql-injection.sql-injection\",\"from\":\"284\",\"to\":\"286\",\"severity\":\"ERROR\"}]},{\"fileMD5\":\"b2c3d4e5f6a1\",\"path\":\"src/main/java/org/apache/commons/lang3/Validate.java\",\"issues\":[{\"ruleID\":\"java.lang.security.audit.hardcoded-secret\",\"from\":\"95\",\"to\":\"95\",\"severity\":\"ERROR\"}]}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Security analysis completed successfully\"}}\"\xb9\x04\n\x16\x43omponentIssueResponse\x12=\n\tcomponent\x18\x01 \x01(\x0b\x32*.scanoss.api.semgrep.v2.ComponentIssueInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\xa8\x03\x92\x41\xa4\x03\n\xa1\x03J\x9e\x03{\"component\":{\"purl\":\"pkg:maven/org.apache.commons/commons-lang3\",\"version\":\"3.12.0\",\"requirement\":\"3.12.0\",\"files\":[{\"fileMD5\":\"a1b2c3d4e5f6\",\"path\":\"src/main/java/org/apache/commons/lang3/StringUtils.java\",\"issues\":[{\"ruleID\":\"java.lang.security.audit.sql-injection.sql-injection\",\"from\":\"284\",\"to\":\"286\",\"severity\":\"ERROR\"}]}]},\"status\":{\"status\":\"SUCCESS\",\"message\":\"Security analysis completed successfully\"}}2\x89\x04\n\x07Semgrep\x12l\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\x1b\x82\xd3\xe4\x93\x02\x15\"\x10/v2/semgrep/echo:\x01*\x12]\n\tGetIssues\x12\".scanoss.api.common.v2.PurlRequest\x1a\'.scanoss.api.semgrep.v2.SemgrepResponse\"\x03\x88\x02\x01\x12\x9a\x01\n\x13GetComponentsIssues\x12(.scanoss.api.common.v2.ComponentsRequest\x1a/.scanoss.api.semgrep.v2.ComponentsIssueResponse\"(\x82\xd3\xe4\x93\x02\"\"\x1d/v2/semgrep/issues/components:\x01*\x12\x93\x01\n\x12GetComponentIssues\x12\'.scanoss.api.common.v2.ComponentRequest\x1a..scanoss.api.semgrep.v2.ComponentIssueResponse\"$\x82\xd3\xe4\x93\x02\x1e\x12\x1c/v2/semgrep/issues/componentB\x85\x02Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\x92\x41\xd0\x01\x12j\n\x17SCANOSS Semgrep Service\"J\n\x0fscanoss-semgrep\x12\"https://github.com/scanoss/semgrep\x1a\x13support@scanoss.com2\x03\x32.0*\x01\x01\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -35,18 +35,34 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'Z/github.com/scanoss/papi/api/semgrepv2;semgrepv2\222A\320\001\022j\n\027SCANOSS Semgrep Service\"J\n\017scanoss-semgrep\022\"https://github.com/scanoss/semgrep\032\023support@scanoss.com2\0032.0*\001\0012\020application/json:\020application/jsonR;\n\003404\0224\n*Returned when the resource does not exist.\022\006\n\004\232\002\001\007' + _globals['_SEMGREPRESPONSE']._loaded_options = None + _globals['_SEMGREPRESPONSE']._serialized_options = b'\030\001' + _globals['_COMPONENTSISSUERESPONSE']._loaded_options = None + _globals['_COMPONENTSISSUERESPONSE']._serialized_options = b'\222A\317\005\n\314\005J\311\005{\"components\":[{\"purl\":\"pkg:maven/org.apache.commons/commons-lang3\",\"version\":\"3.12.0\",\"requirement\":\"3.12.0\",\"files\":[{\"fileMD5\":\"a1b2c3d4e5f6\",\"path\":\"src/main/java/org/apache/commons/lang3/StringUtils.java\",\"issues\":[{\"ruleID\":\"java.lang.security.audit.crypto.weak-hash\",\"from\":\"156\",\"to\":\"159\",\"severity\":\"WARNING\"},{\"ruleID\":\"java.lang.security.audit.sql-injection.sql-injection\",\"from\":\"284\",\"to\":\"286\",\"severity\":\"ERROR\"}]},{\"fileMD5\":\"b2c3d4e5f6a1\",\"path\":\"src/main/java/org/apache/commons/lang3/Validate.java\",\"issues\":[{\"ruleID\":\"java.lang.security.audit.hardcoded-secret\",\"from\":\"95\",\"to\":\"95\",\"severity\":\"ERROR\"}]}]}],\"status\":{\"status\":\"SUCCESS\",\"message\":\"Security analysis completed successfully\"}}' + _globals['_COMPONENTISSUERESPONSE']._loaded_options = None + _globals['_COMPONENTISSUERESPONSE']._serialized_options = b'\222A\244\003\n\241\003J\236\003{\"component\":{\"purl\":\"pkg:maven/org.apache.commons/commons-lang3\",\"version\":\"3.12.0\",\"requirement\":\"3.12.0\",\"files\":[{\"fileMD5\":\"a1b2c3d4e5f6\",\"path\":\"src/main/java/org/apache/commons/lang3/StringUtils.java\",\"issues\":[{\"ruleID\":\"java.lang.security.audit.sql-injection.sql-injection\",\"from\":\"284\",\"to\":\"286\",\"severity\":\"ERROR\"}]}]},\"status\":{\"status\":\"SUCCESS\",\"message\":\"Security analysis completed successfully\"}}' _globals['_SEMGREP'].methods_by_name['Echo']._loaded_options = None - _globals['_SEMGREP'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\031\"\024/api/v2/semgrep/echo:\001*' + _globals['_SEMGREP'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\025\"\020/v2/semgrep/echo:\001*' _globals['_SEMGREP'].methods_by_name['GetIssues']._loaded_options = None - _globals['_SEMGREP'].methods_by_name['GetIssues']._serialized_options = b'\202\323\344\223\002\033\"\026/api/v2/semgrep/issues:\001*' - _globals['_SEMGREPRESPONSE']._serialized_start=195 - _globals['_SEMGREPRESPONSE']._serialized_end=601 - _globals['_SEMGREPRESPONSE_ISSUE']._serialized_start=331 - _globals['_SEMGREPRESPONSE_ISSUE']._serialized_end=398 - _globals['_SEMGREPRESPONSE_FILE']._serialized_start=400 - _globals['_SEMGREPRESPONSE_FILE']._serialized_end=500 - _globals['_SEMGREPRESPONSE_PURLS']._serialized_start=502 - _globals['_SEMGREPRESPONSE_PURLS']._serialized_end=601 - _globals['_SEMGREP']._serialized_start=604 - _globals['_SEMGREP']._serialized_end=852 + _globals['_SEMGREP'].methods_by_name['GetIssues']._serialized_options = b'\210\002\001' + _globals['_SEMGREP'].methods_by_name['GetComponentsIssues']._loaded_options = None + _globals['_SEMGREP'].methods_by_name['GetComponentsIssues']._serialized_options = b'\202\323\344\223\002\"\"\035/v2/semgrep/issues/components:\001*' + _globals['_SEMGREP'].methods_by_name['GetComponentIssues']._loaded_options = None + _globals['_SEMGREP'].methods_by_name['GetComponentIssues']._serialized_options = b'\202\323\344\223\002\036\022\034/v2/semgrep/issues/component' + _globals['_ISSUE']._serialized_start=194 + _globals['_ISSUE']._serialized_end=261 + _globals['_FILE']._serialized_start=263 + _globals['_FILE']._serialized_end=347 + _globals['_SEMGREPRESPONSE']._serialized_start=350 + _globals['_SEMGREPRESPONSE']._serialized_end=573 + _globals['_SEMGREPRESPONSE_PURLS']._serialized_start=486 + _globals['_SEMGREPRESPONSE_PURLS']._serialized_end=569 + _globals['_COMPONENTISSUEINFO']._serialized_start=575 + _globals['_COMPONENTISSUEINFO']._serialized_end=692 + _globals['_COMPONENTSISSUERESPONSE']._serialized_start=695 + _globals['_COMPONENTSISSUERESPONSE']._serialized_end=1565 + _globals['_COMPONENTISSUERESPONSE']._serialized_start=1568 + _globals['_COMPONENTISSUERESPONSE']._serialized_end=2137 + _globals['_SEMGREP']._serialized_start=2140 + _globals['_SEMGREP']._serialized_end=2661 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py index fdda3109..2b7e6c10 100644 --- a/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py +++ b/src/scanoss/api/semgrep/v2/scanoss_semgrep_pb2_grpc.py @@ -27,8 +27,8 @@ class SemgrepStub(object): - """ - Expose all of the SCANOSS Cryptography RPCs here + """* + Expose all of the SCANOSS Semgrep Security Analysis RPCs here """ def __init__(self, channel): @@ -47,22 +47,52 @@ def __init__(self, channel): request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.SerializeToString, response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.FromString, _registered_method=True) + self.GetComponentsIssues = channel.unary_unary( + '/scanoss.api.semgrep.v2.Semgrep/GetComponentsIssues', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.ComponentsIssueResponse.FromString, + _registered_method=True) + self.GetComponentIssues = channel.unary_unary( + '/scanoss.api.semgrep.v2.Semgrep/GetComponentIssues', + request_serializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + response_deserializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.ComponentIssueResponse.FromString, + _registered_method=True) class SemgrepServicer(object): - """ - Expose all of the SCANOSS Cryptography RPCs here + """* + Expose all of the SCANOSS Semgrep Security Analysis RPCs here """ def Echo(self, request, context): - """Standard echo + """Standard health check endpoint to verify service availability and connectivity """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetIssues(self, request, context): - """Get Potential issues associated with a list of PURLs + """[DEPRECATED] Get potential security issues associated with a list of PURLs + This method accepts PURL-based requests and is deprecated in favor of GetComponentsIssues + which accepts ComponentsRequest for better component identification + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentsIssues(self, request, context): + """Get potential security issues associated with multiple components + This is the current method that accepts ComponentsRequest for enhanced component identification + Replaces the deprecated GetIssues method + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetComponentIssues(self, request, context): + """Get potential security issues associated with a single component + This is the current method that accepts ComponentRequest for enhanced component identification + Replaces the deprecated GetIssues method for single component queries """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -81,6 +111,16 @@ def add_SemgrepServicer_to_server(servicer, server): request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.PurlRequest.FromString, response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.SemgrepResponse.SerializeToString, ), + 'GetComponentsIssues': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentsIssues, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.FromString, + response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.ComponentsIssueResponse.SerializeToString, + ), + 'GetComponentIssues': grpc.unary_unary_rpc_method_handler( + servicer.GetComponentIssues, + request_deserializer=scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.FromString, + response_serializer=scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.ComponentIssueResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'scanoss.api.semgrep.v2.Semgrep', rpc_method_handlers) @@ -90,8 +130,8 @@ def add_SemgrepServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. class Semgrep(object): - """ - Expose all of the SCANOSS Cryptography RPCs here + """* + Expose all of the SCANOSS Semgrep Security Analysis RPCs here """ @staticmethod @@ -147,3 +187,57 @@ def GetIssues(request, timeout, metadata, _registered_method=True) + + @staticmethod + def GetComponentsIssues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.semgrep.v2.Semgrep/GetComponentsIssues', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentsRequest.SerializeToString, + scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.ComponentsIssueResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetComponentIssues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/scanoss.api.semgrep.v2.Semgrep/GetComponentIssues', + scanoss_dot_api_dot_common_dot_v2_dot_scanoss__common__pb2.ComponentRequest.SerializeToString, + scanoss_dot_api_dot_semgrep_dot_v2_dot_scanoss__semgrep__pb2.ComponentIssueResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py index 7608af1d..4a1fc7dc 100644 --- a/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py +++ b/src/scanoss/api/vulnerabilities/v2/scanoss_vulnerabilities_pb2.py @@ -27,7 +27,7 @@ from protoc_gen_openapiv2.options import annotations_pb2 as protoc__gen__openapiv2_dot_options_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}\"\xbd\t\n\x1f\x43omponentsVulnerabilityResponse\x12N\n\ncomponents\x18\x01 \x03(\x0b\x32:.scanoss.api.vulnerabilities.v2.ComponentVulnerabilityInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x92\x08\x92\x41\x8e\x08\n\x8b\x08J\x88\x08{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \"1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\",\"requirement\": \"v1.30.0\",\"version\": \"v1.30.0\", \"vulnerabilities\": [{\"id\": \"CVE-2024-54321\", \"cve\": \"CVE-2024-54321\", \"url\": \"https://nvd.nist.gov/vuln/detail/CVE-2024-54321\", \"summary\": \"Denial of service vulnerability\", \"severity\": \"Medium\", \"published\": \"2024-01-15\", \"modified\": \"2024-02-01\", \"source\": \"NDV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L\", \"cvss_score\": 4.3, \"cvss_severity\": \"Medium\"}]}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}2\xc7\x08\n\x0fVulnerabilities\x12x\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"\'\x82\xd3\xe4\x93\x02!\"\x1c/api/v2/vulnerabilities/echo:\x01*\x12q\n\x07GetCpes\x12\x34.scanoss.api.vulnerabilities.v2.VulnerabilityRequest\x1a+.scanoss.api.vulnerabilities.v2.CpeResponse\"\x03\x88\x02\x01\x12\xa2\x01\n\x10GetComponentCpes\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x35.scanoss.api.vulnerabilities.v2.ComponentCpesResponse\".\x82\xd3\xe4\x93\x02(\x12&/api/v2/vulnerabilities/cpes/component\x12\xa9\x01\n\x11GetComponentsCpes\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x36.scanoss.api.vulnerabilities.v2.ComponentsCpesResponse\"2\x82\xd3\xe4\x93\x02,\"\'/api/v2/vulnerabilities/cpes/components:\x01*\x12\x86\x01\n\x12GetVulnerabilities\x12\x34.scanoss.api.vulnerabilities.v2.VulnerabilityRequest\x1a\x35.scanoss.api.vulnerabilities.v2.VulnerabilityResponse\"\x03\x88\x02\x01\x12\xb1\x01\n\x1bGetComponentVulnerabilities\x12\'.scanoss.api.common.v2.ComponentRequest\x1a>.scanoss.api.vulnerabilities.v2.ComponentVulnerabilityResponse\")\x82\xd3\xe4\x93\x02#\x12!/api/v2/vulnerabilities/component\x12\xb8\x01\n\x1cGetComponentsVulnerabilities\x12(.scanoss.api.common.v2.ComponentsRequest\x1a?.scanoss.api.vulnerabilities.v2.ComponentsVulnerabilityResponse\"-\x82\xd3\xe4\x93\x02\'\"\"/api/v2/vulnerabilities/components:\x01*B\x92\x03Z?github.com/scanoss/papi/api/vulnerabilitiesv2;vulnerabilitiesv2\x92\x41\xcd\x02\x12\xd4\x01\n\x1dSCANOSS Vulnerability Service\x12RVulnerability service provides vulnerability intelligence for software components.\"Z\n\x17scanoss-vulnerabilities\x12*https://github.com/scanoss/vulnerabilities\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x01\x02\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}\"\xbd\t\n\x1f\x43omponentsVulnerabilityResponse\x12N\n\ncomponents\x18\x01 \x03(\x0b\x32:.scanoss.api.vulnerabilities.v2.ComponentVulnerabilityInfo\x12\x35\n\x06status\x18\x02 \x01(\x0b\x32%.scanoss.api.common.v2.StatusResponse:\x92\x08\x92\x41\x8e\x08\n\x8b\x08J\x88\x08{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \"1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\",\"requirement\": \"v1.30.0\",\"version\": \"v1.30.0\", \"vulnerabilities\": [{\"id\": \"CVE-2024-54321\", \"cve\": \"CVE-2024-54321\", \"url\": \"https://nvd.nist.gov/vuln/detail/CVE-2024-54321\", \"summary\": \"Denial of service vulnerability\", \"severity\": \"Medium\", \"published\": \"2024-01-15\", \"modified\": \"2024-02-01\", \"source\": \"NDV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L\", \"cvss_score\": 4.3, \"cvss_severity\": \"Medium\"}]}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}2\xb3\x08\n\x0fVulnerabilities\x12t\n\x04\x45\x63ho\x12\".scanoss.api.common.v2.EchoRequest\x1a#.scanoss.api.common.v2.EchoResponse\"#\x82\xd3\xe4\x93\x02\x1d\"\x18/v2/vulnerabilities/echo:\x01*\x12q\n\x07GetCpes\x12\x34.scanoss.api.vulnerabilities.v2.VulnerabilityRequest\x1a+.scanoss.api.vulnerabilities.v2.CpeResponse\"\x03\x88\x02\x01\x12\x9e\x01\n\x10GetComponentCpes\x12\'.scanoss.api.common.v2.ComponentRequest\x1a\x35.scanoss.api.vulnerabilities.v2.ComponentCpesResponse\"*\x82\xd3\xe4\x93\x02$\x12\"/v2/vulnerabilities/cpes/component\x12\xa5\x01\n\x11GetComponentsCpes\x12(.scanoss.api.common.v2.ComponentsRequest\x1a\x36.scanoss.api.vulnerabilities.v2.ComponentsCpesResponse\".\x82\xd3\xe4\x93\x02(\"#/v2/vulnerabilities/cpes/components:\x01*\x12\x86\x01\n\x12GetVulnerabilities\x12\x34.scanoss.api.vulnerabilities.v2.VulnerabilityRequest\x1a\x35.scanoss.api.vulnerabilities.v2.VulnerabilityResponse\"\x03\x88\x02\x01\x12\xad\x01\n\x1bGetComponentVulnerabilities\x12\'.scanoss.api.common.v2.ComponentRequest\x1a>.scanoss.api.vulnerabilities.v2.ComponentVulnerabilityResponse\"%\x82\xd3\xe4\x93\x02\x1f\x12\x1d/v2/vulnerabilities/component\x12\xb4\x01\n\x1cGetComponentsVulnerabilities\x12(.scanoss.api.common.v2.ComponentsRequest\x1a?.scanoss.api.vulnerabilities.v2.ComponentsVulnerabilityResponse\")\x82\xd3\xe4\x93\x02#\"\x1e/v2/vulnerabilities/components:\x01*B\x92\x03Z?github.com/scanoss/papi/api/vulnerabilitiesv2;vulnerabilitiesv2\x92\x41\xcd\x02\x12\xd4\x01\n\x1dSCANOSS Vulnerability Service\x12RVulnerability service provides vulnerability intelligence for software components.\"Z\n\x17scanoss-vulnerabilities\x12*https://github.com/scanoss/vulnerabilities\x1a\x13support@scanoss.com2\x03\x32.0\x1a\x0f\x61pi.scanoss.com*\x02\x01\x02\x32\x10\x61pplication/json:\x10\x61pplication/jsonR;\n\x03\x34\x30\x34\x12\x34\n*Returned when the resource does not exist.\x12\x06\n\x04\x9a\x02\x01\x07\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -50,19 +50,19 @@ _globals['_COMPONENTSVULNERABILITYRESPONSE']._loaded_options = None _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_options = b'\222A\216\010\n\213\010J\210\010{\"components\":[{\"purl\": \"pkg:github/scanoss/engine\", \"requirement\": \"1.0.0\", \"version\": \"1.0.0\", \"vulnerabilities\": [{\"id\": \"DLA-2640-1\", \"cve\": \"DLA-2640-1\", \"url\": \"https://osv.dev/vulnerability/DLA-2640-1\", \"summary\": \"gst-plugins-good1.0 - security update\", \"severity\": \"Critical\", \"published\": \"2021-04-26\", \"modified\": \"2025-05-26\", \"source\": \"OSV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\", \"cvss_score\": 9.8, \"cvss_severity\": \"Critical\"}]}]}, {\"purl\": \"pkg:github/scanoss/scanoss.py\",\"requirement\": \"v1.30.0\",\"version\": \"v1.30.0\", \"vulnerabilities\": [{\"id\": \"CVE-2024-54321\", \"cve\": \"CVE-2024-54321\", \"url\": \"https://nvd.nist.gov/vuln/detail/CVE-2024-54321\", \"summary\": \"Denial of service vulnerability\", \"severity\": \"Medium\", \"published\": \"2024-01-15\", \"modified\": \"2024-02-01\", \"source\": \"NDV\", \"cvss\": [{\"cvss\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:L\", \"cvss_score\": 4.3, \"cvss_severity\": \"Medium\"}]}]}], \"status\": {\"status\": \"SUCCESS\", \"message\": \"Vulnerabilities Successfully retrieved\"}}' _globals['_VULNERABILITIES'].methods_by_name['Echo']._loaded_options = None - _globals['_VULNERABILITIES'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002!\"\034/api/v2/vulnerabilities/echo:\001*' + _globals['_VULNERABILITIES'].methods_by_name['Echo']._serialized_options = b'\202\323\344\223\002\035\"\030/v2/vulnerabilities/echo:\001*' _globals['_VULNERABILITIES'].methods_by_name['GetCpes']._loaded_options = None _globals['_VULNERABILITIES'].methods_by_name['GetCpes']._serialized_options = b'\210\002\001' _globals['_VULNERABILITIES'].methods_by_name['GetComponentCpes']._loaded_options = None - _globals['_VULNERABILITIES'].methods_by_name['GetComponentCpes']._serialized_options = b'\202\323\344\223\002(\022&/api/v2/vulnerabilities/cpes/component' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentCpes']._serialized_options = b'\202\323\344\223\002$\022\"/v2/vulnerabilities/cpes/component' _globals['_VULNERABILITIES'].methods_by_name['GetComponentsCpes']._loaded_options = None - _globals['_VULNERABILITIES'].methods_by_name['GetComponentsCpes']._serialized_options = b'\202\323\344\223\002,\"\'/api/v2/vulnerabilities/cpes/components:\001*' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentsCpes']._serialized_options = b'\202\323\344\223\002(\"#/v2/vulnerabilities/cpes/components:\001*' _globals['_VULNERABILITIES'].methods_by_name['GetVulnerabilities']._loaded_options = None _globals['_VULNERABILITIES'].methods_by_name['GetVulnerabilities']._serialized_options = b'\210\002\001' _globals['_VULNERABILITIES'].methods_by_name['GetComponentVulnerabilities']._loaded_options = None - _globals['_VULNERABILITIES'].methods_by_name['GetComponentVulnerabilities']._serialized_options = b'\202\323\344\223\002#\022!/api/v2/vulnerabilities/component' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentVulnerabilities']._serialized_options = b'\202\323\344\223\002\037\022\035/v2/vulnerabilities/component' _globals['_VULNERABILITIES'].methods_by_name['GetComponentsVulnerabilities']._loaded_options = None - _globals['_VULNERABILITIES'].methods_by_name['GetComponentsVulnerabilities']._serialized_options = b'\202\323\344\223\002\'\"\"/api/v2/vulnerabilities/components:\001*' + _globals['_VULNERABILITIES'].methods_by_name['GetComponentsVulnerabilities']._serialized_options = b'\202\323\344\223\002#\"\036/v2/vulnerabilities/components:\001*' _globals['_VULNERABILITYREQUEST']._serialized_start=219 _globals['_VULNERABILITYREQUEST']._serialized_end=364 _globals['_VULNERABILITYREQUEST_PURLS']._serialized_start=318 @@ -78,19 +78,19 @@ _globals['_COMPONENTSCPESRESPONSE']._serialized_start=1030 _globals['_COMPONENTSCPESRESPONSE']._serialized_end=1581 _globals['_CVSS']._serialized_start=1583 - _globals['_CVSS']._serialized_end=1646 - _globals['_VULNERABILITY']._serialized_start=1649 - _globals['_VULNERABILITY']._serialized_end=1842 - _globals['_VULNERABILITYRESPONSE']._serialized_start=1845 - _globals['_VULNERABILITYRESPONSE']._serialized_end=2098 - _globals['_VULNERABILITYRESPONSE_PURLS']._serialized_start=2001 - _globals['_VULNERABILITYRESPONSE_PURLS']._serialized_end=2094 - _globals['_COMPONENTVULNERABILITYINFO']._serialized_start=2101 - _globals['_COMPONENTVULNERABILITYINFO']._serialized_end=2253 - _globals['_COMPONENTVULNERABILITYRESPONSE']._serialized_start=2256 - _globals['_COMPONENTVULNERABILITYRESPONSE']._serialized_end=2995 - _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_start=2998 - _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_end=4211 - _globals['_VULNERABILITIES']._serialized_start=4214 - _globals['_VULNERABILITIES']._serialized_end=5309 + _globals['_CVSS']._serialized_end=1673 + _globals['_VULNERABILITY']._serialized_start=1676 + _globals['_VULNERABILITY']._serialized_end=1869 + _globals['_VULNERABILITYRESPONSE']._serialized_start=1872 + _globals['_VULNERABILITYRESPONSE']._serialized_end=2125 + _globals['_VULNERABILITYRESPONSE_PURLS']._serialized_start=2028 + _globals['_VULNERABILITYRESPONSE_PURLS']._serialized_end=2121 + _globals['_COMPONENTVULNERABILITYINFO']._serialized_start=2128 + _globals['_COMPONENTVULNERABILITYINFO']._serialized_end=2280 + _globals['_COMPONENTVULNERABILITYRESPONSE']._serialized_start=2283 + _globals['_COMPONENTVULNERABILITYRESPONSE']._serialized_end=3022 + _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_start=3025 + _globals['_COMPONENTSVULNERABILITYRESPONSE']._serialized_end=4238 + _globals['_VULNERABILITIES']._serialized_start=4241 + _globals['_VULNERABILITIES']._serialized_end=5316 # @@protoc_insertion_point(module_scope) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 97bed902..d524e705 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -310,7 +310,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Retrieve vulnerabilities for the given components', ) c_vulns.set_defaults(func=comp_vulns) - c_vulns.add_argument('--grpc', action='store_true', help='Enable gRPC support') # Component Sub-command: component licenses c_licenses = comp_sub.add_parser( @@ -319,7 +318,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 description=f'Show License details: {__version__}', help='Retrieve licenses for the given components', ) - c_licenses.add_argument('--grpc', action='store_true', help='Enable gRPC support') c_licenses.set_defaults(func=comp_licenses) # Component Sub-command: component semgrep @@ -949,7 +947,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p.add_argument( '--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)' ) - p.add_argument('--grpc', action='store_true', help='Enable gRPC support') # Global Scan/Fingerprint filter options for p in [p_scan, p_wfp]: @@ -1059,6 +1056,22 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Timeout (in seconds) for syft to complete (optional - default 600)', ) + # gRPC support options + for p in [ + c_vulns, + p_scan, + p_cs, + p_crypto_algorithms, + p_crypto_hints, + p_crypto_versions_in_range, + c_semgrep, + c_provenance, + c_search, + c_versions, + c_licenses, + ]: + p.add_argument('--grpc', action='store_true', help='Enable gRPC support') + # Help/Trace command options for p in [ p_scan, @@ -2178,6 +2191,7 @@ def comp_semgrep(parser, args): pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), + use_grpc=args.grpc, ) if not comps.get_semgrep_details(args.input, args.purl, args.output): sys.exit(1) @@ -2216,6 +2230,7 @@ def comp_search(parser, args): pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), + use_grpc=args.grpc, ) if not comps.search_components( args.output, @@ -2261,6 +2276,7 @@ def comp_versions(parser, args): pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), + use_grpc=args.grpc, ) if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit): sys.exit(1) @@ -2296,6 +2312,7 @@ def comp_provenance(parser, args): pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), + use_grpc=args.grpc, ) if not comps.get_provenance_details(args.input, args.purl, args.output, args.origin): sys.exit(1) diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 25224fbe..6498c1a2 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -77,6 +77,7 @@ def __init__( # noqa: PLR0913, PLR0915 """ super().__init__(debug, trace, quiet) ver_details = Scanner.version_details() + self.use_grpc = use_grpc self.grpc_api = ScanossGrpc( url=grpc_url, debug=debug, @@ -91,7 +92,6 @@ def __init__( # noqa: PLR0913, PLR0915 timeout=timeout, req_headers=req_headers, ignore_cert_errors=ignore_cert_errors, - use_grpc=use_grpc, ) self.cdx = CycloneDx(debug=self.debug) @@ -190,32 +190,6 @@ def _close_file(self, filename: str = None, file: TextIO = None) -> None: self.print_trace(f'Closing file: {filename}') file.close() - def get_crypto_details(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: - """ - Retrieve the cryptographic details for the supplied PURLs - - :param json_file: PURL JSON request file (optional) - :param purls: PURL request array (optional) - :param output_file: output filename (optional). Default: STDOUT - :return: True on success, False otherwise - """ - success = False - purls_request = self.load_purls(json_file, purls) - if purls_request is None or len(purls_request) == 0: - return False - file = self._open_file_or_sdtout(output_file) - if file is None: - return False - self.print_msg('Sending PURLs to Crypto API for decoration...') - response = self.grpc_api.get_crypto_json(purls_request) - if response: - print(json.dumps(response, indent=2, sort_keys=True), file=file) - success = True - if output_file: - self.print_msg(f'Results written to: {output_file}') - self._close_file(output_file, file) - return success - def get_vulnerabilities(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: """ Retrieve any vulnerabilities related to the given PURLs @@ -233,7 +207,7 @@ def get_vulnerabilities(self, json_file: str = None, purls: [] = None, output_fi if file is None: return False self.print_msg('Sending PURLs to Vulnerability API for decoration...') - response = self.grpc_api.get_vulnerabilities_json(purls_request) + response = self.grpc_api.get_vulnerabilities_json(purls_request, use_grpc=self.use_grpc) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) success = True @@ -252,14 +226,14 @@ def get_semgrep_details(self, json_file: str = None, purls: [] = None, output_fi :return: True on success, False otherwise """ success = False - purls_request = self.load_purls(json_file, purls) + purls_request = self.load_comps(json_file, purls) if purls_request is None or len(purls_request) == 0: return False file = self._open_file_or_sdtout(output_file) if file is None: return False self.print_msg('Sending PURLs to Semgrep API for decoration...') - response = self.grpc_api.get_semgrep_json(purls_request) + response = self.grpc_api.get_semgrep_json(purls_request, use_grpc=self.use_grpc) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) success = True @@ -309,7 +283,7 @@ def search_components( # noqa: PLR0913, PLR0915 if file is None: return False self.print_msg('Sending search data to Components API...') - response = self.grpc_api.search_components_json(request) + response = self.grpc_api.search_components_json(request, use_grpc=self.use_grpc) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) success = True @@ -345,7 +319,7 @@ def get_component_versions( if file is None: return False self.print_msg('Sending PURLs to Component Versions API...') - response = self.grpc_api.get_component_versions_json(request) + response = self.grpc_api.get_component_versions_json(request, use_grpc=self.use_grpc) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) success = True @@ -370,7 +344,7 @@ def get_provenance_details( bool: True on success, False otherwise """ success = False - purls_request = self.load_purls(json_file, purls) + purls_request = self.load_comps(json_file, purls) if purls_request is None or len(purls_request) == 0: return False file = self._open_file_or_sdtout(output_file) @@ -378,10 +352,10 @@ def get_provenance_details( return False if origin: self.print_msg('Sending PURLs to Geo Provenance Origin API for decoration...') - response = self.grpc_api.get_provenance_origin(purls_request) + response = self.grpc_api.get_provenance_origin(purls_request, use_grpc=self.use_grpc) else: self.print_msg('Sending PURLs to Geo Provenance Declared API for decoration...') - response = self.grpc_api.get_provenance_json(purls_request) + response = self.grpc_api.get_provenance_json(purls_request, use_grpc=self.use_grpc) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) success = True @@ -413,7 +387,7 @@ def get_licenses(self, json_file: str = None, purls: [] = None, output_file: str # We'll use the new ComponentBatchRequest instead of deprecated PurlRequest for the license api component_batch_request = {'components': purls_request.get('purls')} - response = self.grpc_api.get_licenses(component_batch_request) + response = self.grpc_api.get_licenses(component_batch_request, use_grpc=self.use_grpc) if response: print(json.dumps(response, indent=2, sort_keys=True), file=file) success = True diff --git a/src/scanoss/cryptography.py b/src/scanoss/cryptography.py index 7957609f..5ab91fea 100644 --- a/src/scanoss/cryptography.py +++ b/src/scanoss/cryptography.py @@ -19,12 +19,13 @@ class ScanossCryptographyError(Exception): @dataclass class CryptographyConfig: purl: List[str] + debug: bool = False + header: Optional[str] = None input_file: Optional[str] = None output_file: Optional[str] = None - header: Optional[str] = None - debug: bool = False - trace: bool = False quiet: bool = False + trace: bool = False + use_grpc: bool = False with_range: bool = False def _process_input_file(self) -> dict: @@ -69,7 +70,7 @@ def __post_init__(self): parts = purl.split('@') if not (len(parts) >= MIN_SPLIT_PARTS and parts[1]): raise ScanossCryptographyError( - f'Invalid PURL format: "{purl}".' f'It must include a version (e.g., pkg:type/name@version)' + f'Invalid PURL format: "{purl}".It must include a version (e.g., pkg:type/name@version)' ) if self.input_file: purl_request = self._process_input_file() @@ -91,13 +92,14 @@ def __post_init__(self): def create_cryptography_config_from_args(args) -> CryptographyConfig: return CryptographyConfig( debug=getattr(args, 'debug', False), - trace=getattr(args, 'trace', False), - quiet=getattr(args, 'quiet', False), - with_range=getattr(args, 'with_range', False), - purl=getattr(args, 'purl', []), + header=getattr(args, 'header', None), input_file=getattr(args, 'input', None), output_file=getattr(args, 'output', None), - header=getattr(args, 'header', None), + purl=getattr(args, 'purl', []), + quiet=getattr(args, 'quiet', False), + trace=getattr(args, 'trace', False), + use_grpc=getattr(args, 'grpc', False), + with_range=getattr(args, 'with_range', False), ) @@ -134,7 +136,7 @@ def __init__( self.client = client self.config = config - self.purls_request = self._build_purls_request() + self.components_request = self._build_components_request() self.results = None def get_algorithms(self) -> Optional[Dict]: @@ -145,15 +147,16 @@ def get_algorithms(self) -> Optional[Dict]: Optional[Dict]: The folder hash response from the gRPC client, or None if an error occurs. """ - if not self.purls_request: + if not self.components_request or not self.components_request.get('components'): raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') - self.base.print_stderr( - f'Getting cryptographic algorithms for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' - ) + components_str = ', '.join(p['purl'] for p in self.components_request['components']) + self.base.print_stderr(f'Getting cryptographic algorithms for {components_str}') if self.config.with_range: - response = self.client.get_crypto_algorithms_in_range_for_purl(self.purls_request) + response = self.client.get_crypto_algorithms_in_range_for_purl( + self.components_request, self.config.use_grpc + ) else: - response = self.client.get_crypto_algorithms_for_purl(self.purls_request) + response = self.client.get_crypto_algorithms_for_purl(self.components_request, self.config.use_grpc) if response: self.results = response @@ -167,17 +170,17 @@ def get_encryption_hints(self) -> Optional[Dict]: Optional[Dict]: The encryption hints response from the gRPC client, or None if an error occurs. """ - if not self.purls_request: + if not self.components_request or not self.components_request.get('components'): raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') self.base.print_stderr( f'Getting encryption hints ' f'{"in range" if self.config.with_range else ""} ' - f'for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' + f'for {", ".join([p["purl"] for p in self.components_request["components"]])}' ) if self.config.with_range: - response = self.client.get_encryption_hints_in_range_for_purl(self.purls_request) + response = self.client.get_encryption_hints_in_range_for_purl(self.components_request, self.config.use_grpc) else: - response = self.client.get_encryption_hints_for_purl(self.purls_request) + response = self.client.get_encryption_hints_for_purl(self.components_request, self.config.use_grpc) if response: self.results = response @@ -191,20 +194,21 @@ def get_versions_in_range(self) -> Optional[Dict]: Optional[Dict]: The versions in range response from the gRPC client, or None if an error occurs. """ - if not self.purls_request: + if not self.components_request or not self.components_request.get('components'): raise ScanossCryptographyError('No PURLs supplied. Provide --purl or --input.') - self.base.print_stderr( - f'Getting versions in range for {", ".join([p["purl"] for p in self.purls_request["purls"]])}' - ) + components_str = ', '.join(p['purl'] for p in self.components_request['components']) + self.base.print_stderr(f'Getting versions in range for {components_str}') - response = self.client.get_versions_in_range_for_purl(self.purls_request) + response = self.client.get_versions_in_range_for_purl(self.components_request, self.config.use_grpc) if response: self.results = response return self.results - def _build_purls_request(self) -> Optional[dict]: + def _build_components_request( + self, + ) -> Optional[dict]: """ Load the specified purls from a JSON file or a list of PURLs and return a dictionary @@ -212,7 +216,7 @@ def _build_purls_request(self) -> Optional[dict]: Optional[dict]: The dictionary containing the PURLs """ return { - 'purls': [ + 'components': [ { 'purl': self._remove_version_from_purl(purl), 'requirement': self._extract_version_from_purl(purl), diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index deed00b6..5cb73978 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -24,14 +24,13 @@ import concurrent.futures import http.client as http_client -import json import logging import os import sys import time import uuid from dataclasses import dataclass -from enum import IntEnum +from enum import Enum, IntEnum from typing import Dict, Optional from urllib.parse import urlparse @@ -44,7 +43,6 @@ from pypac.resolver import ProxyResolver from urllib3.exceptions import InsecureRequestWarning -from scanoss.api.licenses.v2.scanoss_licenses_pb2 import ComponentsLicenseResponse from scanoss.api.licenses.v2.scanoss_licenses_pb2_grpc import LicenseStub from scanoss.api.scanning.v2.scanoss_scanning_pb2_grpc import ScanningStub from scanoss.constants import DEFAULT_TIMEOUT @@ -53,29 +51,20 @@ from .api.common.v2.scanoss_common_pb2 import ( ComponentsRequest, EchoRequest, - EchoResponse, - PurlRequest, StatusCode, StatusResponse, ) from .api.components.v2.scanoss_components_pb2 import ( CompSearchRequest, - CompSearchResponse, CompVersionRequest, - CompVersionResponse, ) from .api.components.v2.scanoss_components_pb2_grpc import ComponentsStub from .api.cryptography.v2.scanoss_cryptography_pb2_grpc import CryptographyStub from .api.dependencies.v2.scanoss_dependencies_pb2 import DependencyRequest from .api.dependencies.v2.scanoss_dependencies_pb2_grpc import DependenciesStub -from .api.geoprovenance.v2.scanoss_geoprovenance_pb2 import ContributorResponse from .api.geoprovenance.v2.scanoss_geoprovenance_pb2_grpc import GeoProvenanceStub from .api.scanning.v2.scanoss_scanning_pb2 import HFHRequest -from .api.semgrep.v2.scanoss_semgrep_pb2 import SemgrepResponse from .api.semgrep.v2.scanoss_semgrep_pb2_grpc import SemgrepStub -from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2 import ( - ComponentsVulnerabilityResponse, -) from .api.vulnerabilities.v2.scanoss_vulnerabilities_pb2_grpc import VulnerabilitiesStub from .scanossbase import ScanossBase @@ -87,6 +76,35 @@ MAX_CONCURRENT_REQUESTS = 5 # Maximum number of concurrent requests to make +# REST API endpoint mappings with HTTP methods +REST_ENDPOINTS = { + 'vulnerabilities.GetComponentsVulnerabilities': {'path': '/vulnerabilities/components', 'method': 'POST'}, + 'dependencies.Echo': {'path': '/dependencies/echo', 'method': 'POST'}, + 'dependencies.GetDependencies': {'path': '/dependencies/dependencies', 'method': 'POST'}, + 'cryptography.Echo': {'path': '/cryptography/echo', 'method': 'POST'}, + 'cryptography.GetComponentsAlgorithms': {'path': '/cryptography/algorithms/components', 'method': 'POST'}, + 'cryptography.GetComponentsAlgorithmsInRange': { + 'path': '/cryptography/algorithms/range/components', + 'method': 'POST', + }, + 'cryptography.GetComponentsEncryptionHints': {'path': '/cryptography/hints/components', 'method': 'POST'}, + 'cryptography.GetComponentsHintsInRange': {'path': '/cryptography/hints/components/range', 'method': 'POST'}, + 'cryptography.GetComponentsVersionsInRange': { + 'path': '/cryptography/algorithms/versions/range/components', + 'method': 'POST', + }, + 'components.SearchComponents': {'path': '/components/search', 'method': 'GET'}, + 'components.GetComponentVersions': {'path': '/components/versions', 'method': 'GET'}, + 'geoprovenance.GetCountryContributorsByComponents': { + 'path': '/geoprovenance/countries/components', + 'method': 'POST', + }, + 'geoprovenance.GetOriginByComponents': {'path': '/geoprovenance/origin/components', 'method': 'POST'}, + 'licenses.GetComponentsLicenses': {'path': '/licenses/components', 'method': 'POST'}, + 'semgrep.GetComponentsIssues': {'path': '/semgrep/issues/components', 'method': 'POST'}, + 'scanning.FolderHashScan': {'path': '/scanning/hfh/scan', 'method': 'POST'}, +} + class ScanossGrpcError(Exception): """ @@ -99,12 +117,23 @@ class ScanossGrpcError(Exception): class ScanossGrpcStatusCode(IntEnum): """Status codes for SCANOSS gRPC responses""" + UNSPECIFIED = 0 SUCCESS = 1 - SUCCESS_WITH_WARNINGS = 2 - FAILED_WITH_WARNINGS = 3 + SUCCEEDED_WITH_WARNINGS = 2 + WARNING = 3 FAILED = 4 +class ScanossRESTStatusCode(Enum): + """Status codes for SCANOSS REST responses""" + + UNSPECIFIED = 'UNSPECIFIED' + SUCCESS = 'SUCCESS' + SUCCEEDED_WITH_WARNINGS = 'SUCCEEDED_WITH_WARNINGS' + WARNING = 'WARNING' + FAILED = 'FAILED' + + class ScanossGrpc(ScanossBase): """ Client for gRPC functionality @@ -112,20 +141,20 @@ class ScanossGrpc(ScanossBase): def __init__( # noqa: PLR0912, PLR0913, PLR0915 self, - url: str = None, + url: Optional[str] = None, debug: bool = False, trace: bool = False, quiet: bool = False, - ca_cert: str = None, - api_key: str = None, - ver_details: str = None, + ca_cert: Optional[str] = None, + api_key: Optional[str] = None, + ver_details: Optional[str] = None, timeout: int = 600, - proxy: str = None, - grpc_proxy: str = None, - pac: PACFile = None, - req_headers: dict = None, + proxy: Optional[str] = None, + grpc_proxy: Optional[str] = None, + pac: Optional[PACFile] = None, + req_headers: Optional[dict] = None, ignore_cert_errors: bool = False, - use_grpc: bool = False, + use_grpc: Optional[bool] = False, ): """ @@ -234,53 +263,25 @@ def _load_cert(cls, cert_file: str) -> bytes: with open(cert_file, 'rb') as f: return f.read() - def deps_echo(self, message: str = 'Hello there!') -> str: + def deps_echo(self, message: str = 'Hello there!') -> Optional[dict]: """ Send Echo message to the Dependency service :param self: :param message: Message to send (default: Hello there!) :return: echo or None """ - request_id = str(uuid.uuid4()) - resp: EchoResponse - try: - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - resp = self.dependencies_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - return resp.message - self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}') - return None + return self._call_api('dependencies.Echo', self.dependencies_stub.Echo, {'message': message}, EchoRequest) - def crypto_echo(self, message: str = 'Hello there!') -> str: + def crypto_echo(self, message: str = 'Hello there!') -> Optional[dict]: """ Send Echo message to the Cryptography service :param self: :param message: Message to send (default: Hello there!) :return: echo or None """ - request_id = str(uuid.uuid4()) - resp: EchoResponse - try: - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - resp = self.crypto_stub.Echo(EchoRequest(message=message), metadata=metadata, timeout=3) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - return resp.message - self.print_stderr(f'ERROR: Problem sending Echo request ({message}) to {self.url}. rqId: {request_id}') - return None + return self._call_api('cryptography.Echo', self.crypto_stub.Echo, {'message': message}, EchoRequest) - def get_dependencies(self, dependencies: json, depth: int = 1) -> dict: + def get_dependencies(self, dependencies: Optional[dict] = None, depth: int = 1) -> Optional[dict]: if not dependencies: self.print_stderr('ERROR: No dependency data supplied to submit to the API.') return None @@ -289,7 +290,7 @@ def get_dependencies(self, dependencies: json, depth: int = 1) -> dict: self.print_stderr(f'ERROR: No response for dependency request: {dependencies}') return resp - def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: + def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> Optional[dict]: """ Client function to call the rpc for GetDependencies :param dependencies: Message to send to the service @@ -304,11 +305,11 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: self.print_stderr('ERROR: No dependency data supplied to send to decoration service.') return None all_responses = [] - # determine if we are using gRPC or REST based on the use_grpc flag - process_file = self._process_dep_file_grpc if self.use_grpc else self._process_dep_file_rest # Process the dependency files in parallel with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS) as executor: - future_to_file = {executor.submit(process_file, file, depth): file for file in files_json} + future_to_file = { + executor.submit(self._process_dep_file, file, depth, self.use_grpc): file for file in files_json + } for future in concurrent.futures.as_completed(future_to_file): response = future.result() if response: @@ -326,161 +327,104 @@ def get_dependencies_json(self, dependencies: dict, depth: int = 1) -> dict: merged_response['status'] = response['status'] return merged_response - def _process_dep_file_grpc(self, file, depth: int = 1) -> dict: + def _process_dep_file(self, file, depth: int = 1, use_grpc: Optional[bool] = None) -> Optional[dict]: """ - Process a single file using gRPC + Process a single dependency file using either gRPC or REST - :param file: dependency file purls - :param depth: depth to search (default: 1) - :return: response JSON or None - """ - request_id = str(uuid.uuid4()) - try: - file_request = {'files': [file]} - request = ParseDict(file_request, DependencyRequest()) - request.depth = depth - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) - self.print_debug(f'Sending dependency data via gRPC for decoration (rqId: {request_id})...') - resp = self.dependencies_stub.GetDependencies(request, metadata=metadata, timeout=self.timeout) - return MessageToDict(resp, preserving_proto_field_name=True) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - return None + Args: + file: dependency file purls + depth: depth to search (default: 1) + use_grpc: Whether to use gRPC or REST (None = use instance default) - def get_vulnerabilities_json(self, purls: dict) -> dict: - """ - Client function to call the rpc for Vulnerability GetVulnerabilities - It will either use REST (default) or gRPC depending on the use_grpc flag - :param purls: Message to send to the service - :return: Server response or None + Returns: + response JSON or None """ - if self.use_grpc: - return self._get_vulnerabilities_grpc(purls) - else: - return self._get_vulnerabilities_rest(purls) + file_request = {'files': [file], 'depth': depth} + + return self._call_api( + 'dependencies.GetDependencies', + self.dependencies_stub.GetDependencies, + file_request, + DependencyRequest, + 'Sending dependency data for decoration (rqId: {rqId})...', + use_grpc=use_grpc, + ) - def _get_vulnerabilities_grpc(self, purls: dict) -> dict: + def get_vulnerabilities_json(self, purls: Optional[dict] = None, use_grpc: Optional[bool] = None) -> Optional[dict]: """ Client function to call the rpc for Vulnerability GetVulnerabilities - :param purls: Message to send to the service - :return: Server response or None + It will either use REST (default) or gRPC + + Args: + purls (dict): Message to send to the service + + Returns: + Server response or None """ - if not purls: - self.print_stderr('ERROR: No message supplied to send to gRPC service.') - return None - request_id = str(uuid.uuid4()) - resp: ComponentsVulnerabilityResponse - try: - request = ParseDict(purls, ComponentsRequest()) # Parse the JSON/Dict into the purl request object - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending vulnerability data for decoration (rqId: {request_id})...') - resp = self.vuln_stub.GetComponentsVulnerabilities(request, metadata=metadata, timeout=self.timeout) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - if not self._check_status_response(resp.status, request_id): - return None - resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict - del resp_dict['status'] - return resp_dict - return None + return self._call_api( + 'vulnerabilities.GetComponentsVulnerabilities', + self.vuln_stub.GetComponentsVulnerabilities, + purls, + ComponentsRequest, + 'Sending vulnerability data for decoration (rqId: {rqId})...', + use_grpc=use_grpc, + ) - def get_semgrep_json(self, purls: dict) -> dict: + def get_semgrep_json(self, purls: Optional[dict] = None, use_grpc: Optional[bool] = None) -> Optional[dict]: """ Client function to call the rpc for Semgrep GetIssues - :param purls: Message to send to the service - :return: Server response or None + + Args: + purls (dict): Message to send to the service + use_grpc (bool): Whether to use gRPC or REST + + Returns: + Server response or None """ - if not purls: - self.print_stderr('ERROR: No message supplied to send to gRPC service.') - return None - request_id = str(uuid.uuid4()) - resp: SemgrepResponse - try: - request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending semgrep data for decoration (rqId: {request_id})...') - resp = self.semgrep_stub.GetIssues(request, metadata=metadata, timeout=self.timeout) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - if not self._check_status_response(resp.status, request_id): - return None - resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict - del resp_dict['status'] - return resp_dict - return None + return self._call_api( + 'semgrep.GetComponentsIssues', + self.semgrep_stub.GetComponentsIssues, + purls, + ComponentsRequest, + 'Sending semgrep data for decoration (rqId: {rqId})...', + use_grpc=use_grpc, + ) - def search_components_json(self, search: dict) -> dict: + def search_components_json(self, search: dict, use_grpc: Optional[bool] = None) -> Optional[dict]: """ Client function to call the rpc for Components SearchComponents - :param search: Message to send to the service - :return: Server response or None - """ - if not search: - self.print_stderr('ERROR: No message supplied to send to gRPC service.') - return None - request_id = str(uuid.uuid4()) - resp: CompSearchResponse - try: - request = ParseDict(search, CompSearchRequest()) # Parse the JSON/Dict into the purl request object - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending component search data (rqId: {request_id})...') - resp = self.comp_search_stub.SearchComponents(request, metadata=metadata, timeout=self.timeout) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - if not self._check_status_response(resp.status, request_id): - return None - resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict - del resp_dict['status'] - return resp_dict - return None - def get_component_versions_json(self, search: dict) -> dict: + Args: + search (dict): Message to send to the service + Returns: + Server response or None + """ + return self._call_api( + 'components.SearchComponents', + self.comp_search_stub.SearchComponents, + search, + CompSearchRequest, + 'Sending component search data for decoration (rqId: {rqId})...', + use_grpc=use_grpc, + ) + + def get_component_versions_json(self, search: dict, use_grpc: Optional[bool] = None) -> Optional[dict]: """ Client function to call the rpc for Components GetComponentVersions - :param search: Message to send to the service - :return: Server response or None - """ - if not search: - self.print_stderr('ERROR: No message supplied to send to gRPC service.') - return None - request_id = str(uuid.uuid4()) - resp: CompVersionResponse - try: - request = ParseDict(search, CompVersionRequest()) # Parse the JSON/Dict into the purl request object - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending component version data (rqId: {request_id})...') - resp = self.comp_search_stub.GetComponentVersions(request, metadata=metadata, timeout=self.timeout) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - if not self._check_status_response(resp.status, request_id): - return None - resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict - del resp_dict['status'] - return resp_dict - return None + + Args: + search (dict): Message to send to the service + Returns: + Server response or None + """ + return self._call_api( + 'components.GetComponentVersions', + self.comp_search_stub.GetComponentVersions, + search, + CompVersionRequest, + 'Sending component version data for decoration (rqId: {rqId})...', + use_grpc=use_grpc, + ) def folder_hash_scan(self, request: Dict) -> Optional[Dict]: """ @@ -499,6 +443,44 @@ def folder_hash_scan(self, request: Dict) -> Optional[Dict]: 'Sending folder hash scan data (rqId: {rqId})...', ) + def _call_api( + self, + endpoint_key: str, + rpc_method, + request_input, + request_type, + debug_msg: Optional[str] = None, + use_grpc: Optional[bool] = None, + ) -> Optional[Dict]: + """ + Unified method to call either gRPC or REST API based on configuration + + Args: + endpoint_key (str): The key to lookup the REST endpoint in REST_ENDPOINTS + rpc_method: The gRPC stub method (used only if use_grpc is True) + request_input: Either a dict or a gRPC request object + request_type: The type of the gRPC request object (used only if use_grpc is True) + debug_msg (str, optional): Debug message template that can include {rqId} placeholder + use_grpc (bool, optional): Override the instance's use_grpc setting. If None, uses self.use_grpc + + Returns: + dict: The parsed response as a dictionary, or None if something went wrong + """ + if not request_input: + self.print_stderr('ERROR: No message supplied to send to service.') + return None + + # Determine whether to use gRPC or REST + use_grpc_flag = use_grpc if use_grpc is not None else self.use_grpc + + if use_grpc_flag: + return self._call_rpc(rpc_method, request_input, request_type, debug_msg) + else: + # For REST, we only need the dict input + if not isinstance(request_input, dict): + request_input = MessageToDict(request_input, preserving_proto_field_name=True) + return self._call_rest(endpoint_key, request_input, debug_msg) + def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional[str] = None) -> Optional[Dict]: """ Call a gRPC method and return the response as a dictionary @@ -518,20 +500,21 @@ def _call_rpc(self, rpc_method, request_input, request_type, debug_msg: Optional else: request_obj = request_input metadata = self.metadata[:] + [('x-request-id', request_id)] - self.print_debug(debug_msg.format(rqId=request_id)) + if debug_msg: + self.print_debug(debug_msg.format(rqId=request_id)) try: resp = rpc_method(request_obj, metadata=metadata, timeout=self.timeout) except grpc.RpcError as e: raise ScanossGrpcError( f'{e.__class__.__name__} while sending gRPC message (rqId: {request_id}): {e.details()}' ) - if resp and not self._check_status_response(resp.status, request_id): + if resp and not self._check_status_response_grpc(resp.status, request_id): return None resp_dict = MessageToDict(resp, preserving_proto_field_name=True) return resp_dict - def _check_status_response(self, status_response: StatusResponse, request_id: str = None) -> bool: + def _check_status_response_grpc(self, status_response: StatusResponse, request_id: str = None) -> bool: """ Check the response object to see if the command was successful or not :param status_response: Status Response @@ -546,15 +529,59 @@ def _check_status_response(self, status_response: StatusResponse, request_id: st if status_code > ScanossGrpcStatusCode.SUCCESS: ret_val = False # default to failed msg = 'Unsuccessful' - if status_code == ScanossGrpcStatusCode.SUCCESS_WITH_WARNINGS: + if status_code == ScanossGrpcStatusCode.SUCCEEDED_WITH_WARNINGS: msg = 'Succeeded with warnings' ret_val = True # No need to fail as it succeeded with warnings - elif status_code == ScanossGrpcStatusCode.FAILED_WITH_WARNINGS: + elif status_code == ScanossGrpcStatusCode.WARNING: msg = 'Failed with warnings' self.print_stderr(f'{msg} (rqId: {request_id} - status: {status_code}): {status_response.message}') return ret_val return True + def check_status_response_rest(self, status_dict: dict, request_id: Optional[str] = None) -> bool: + """ + Check the REST response dictionary to see if the command was successful or not + + Args: + status_dict (dict): Status dictionary from REST response containing 'status' and 'message' keys + request_id (str, optional): Request ID for logging + Returns: + bool: True if successful, False otherwise + """ + if not status_dict: + self.print_stderr(f'Warning: No status response supplied (rqId: {request_id}). Assuming it was ok.') + return True + + if request_id: + self.print_debug(f'Checking response status (rqId: {request_id}): {status_dict}') + + # Get status from dictionary - it can be either a string or nested dict + status = status_dict.get('status') + message = status_dict.get('message', '') + ret_val = True + + # Handle case where status might be a string directly + if isinstance(status, str): + status_str = status.upper() + if status_str == ScanossRESTStatusCode.SUCCESS.value: + ret_val = True + elif status_str == ScanossRESTStatusCode.SUCCEEDED_WITH_WARNINGS.value: + self.print_stderr(f'Succeeded with warnings (rqId: {request_id}): {message}') + ret_val = True + elif status_str == ScanossRESTStatusCode.WARNING.value: + self.print_stderr(f'Failed with warnings (rqId: {request_id}): {message}') + ret_val = False + elif status_str == ScanossRESTStatusCode.FAILED.value: + self.print_stderr(f'Unsuccessful (rqId: {request_id}): {message}') + ret_val = False + else: + self.print_debug(f'Unknown status "{status_str}" (rqId: {request_id}). Assuming success.') + ret_val = True + + # Otherwise asume success + self.print_debug(f'Unexpected status type {type(status)} (rqId: {request_id}). Assuming success.') + return ret_val + def _get_proxy_config(self): """ Set the grpc_proxy/http_proxy/https_proxy environment variables if PAC file has been specified @@ -577,138 +604,146 @@ def _get_proxy_config(self): os.environ['http_proxy'] = proxies.get('http') or '' os.environ['https_proxy'] = proxies.get('https') or '' - def get_provenance_json(self, purls: dict) -> dict: + def get_provenance_json(self, purls: dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ - Client function to call the rpc for GetComponentProvenance - :param purls: Message to send to the service - :return: Server response or None + Client function to call the rpc for GetComponentContributors + + Args: + purls (dict): ComponentsRequest + use_grpc (bool): Whether to use gRPC or REST (None = use instance default) + + Returns: + dict: JSON response or None """ - if not purls: - self.print_stderr('ERROR: No message supplied to send to gRPC service.') - return None - request_id = str(uuid.uuid4()) - resp: ContributorResponse - try: - request = ParseDict(purls, PurlRequest()) # Parse the JSON/Dict into the purl request object - metadata = self.metadata[:] - metadata.append(('x-request-id', request_id)) # Set a Request ID - self.print_debug(f'Sending data for provenance decoration (rqId: {request_id})...') - resp = self.provenance_stub.GetComponentContributors(request, metadata=metadata, timeout=self.timeout) - except Exception as e: - self.print_stderr( - f'ERROR: {e.__class__.__name__} Problem encountered sending gRPC message (rqId: {request_id}): {e}' - ) - else: - if resp: - if not self._check_status_response(resp.status, request_id): - return None - resp_dict = MessageToDict(resp, preserving_proto_field_name=True) # Convert gRPC response to a dict - return resp_dict - return None + return self._call_api( + 'geoprovenance.GetCountryContributorsByComponents', + self.provenance_stub.GetCountryContributorsByComponents, + purls, + ComponentsRequest, + 'Sending data for provenance decoration (rqId: {rqId})...', + use_grpc=use_grpc, + ) - def get_provenance_origin(self, request: Dict) -> Optional[Dict]: + def get_provenance_origin(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ - Client function to call the rpc for GetComponentOrigin + Client function to call the rpc for GetOriginByComponents Args: - request (Dict): GetComponentOrigin Request + request (Dict): GetOriginByComponents Request Returns: Optional[Dict]: OriginResponse, or None if the request was not successfull """ - return self._call_rpc( - self.provenance_stub.GetComponentOrigin, + return self._call_api( + 'geoprovenance.GetOriginByComponents', + self.provenance_stub.GetOriginByComponents, request, - PurlRequest, + ComponentsRequest, 'Sending data for provenance origin decoration (rqId: {rqId})...', + use_grpc=use_grpc, ) - def get_crypto_algorithms_for_purl(self, request: Dict) -> Optional[Dict]: + def get_crypto_algorithms_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ - Client function to call the rpc for GetAlgorithms for a list of purls + Client function to call the rpc for GetComponentsAlgorithms for a list of purls Args: - request (Dict): PurlRequest + request (Dict): ComponentsRequest + use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default) Returns: Optional[Dict]: AlgorithmResponse, or None if the request was not successfull """ - return self._call_rpc( - self.crypto_stub.GetAlgorithms, + return self._call_api( + 'cryptography.GetComponentsAlgorithms', + self.crypto_stub.GetComponentsAlgorithms, request, - PurlRequest, + ComponentsRequest, 'Sending data for cryptographic algorithms decoration (rqId: {rqId})...', + use_grpc=use_grpc, ) - def get_crypto_algorithms_in_range_for_purl(self, request: Dict) -> Optional[Dict]: + def get_crypto_algorithms_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ - Client function to call the rpc for GetAlgorithmsInRange for a list of purls + Client function to call the rpc for GetComponentsAlgorithmsInRange for a list of purls Args: - request (Dict): PurlRequest + request (Dict): ComponentsRequest + use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default) Returns: Optional[Dict]: AlgorithmsInRangeResponse, or None if the request was not successfull """ - return self._call_rpc( - self.crypto_stub.GetAlgorithmsInRange, + return self._call_api( + 'cryptography.GetComponentsAlgorithmsInRange', + self.crypto_stub.GetComponentsAlgorithmsInRange, request, - PurlRequest, + ComponentsRequest, 'Sending data for cryptographic algorithms in range decoration (rqId: {rqId})...', + use_grpc=use_grpc, ) - def get_encryption_hints_for_purl(self, request: Dict) -> Optional[Dict]: + def get_encryption_hints_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ - Client function to call the rpc for GetEncryptionHints for a list of purls + Client function to call the rpc for GetComponentsEncryptionHints for a list of purls Args: - request (Dict): PurlRequest + request (Dict): ComponentsRequest + use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default) Returns: Optional[Dict]: HintsResponse, or None if the request was not successfull """ - return self._call_rpc( - self.crypto_stub.GetEncryptionHints, + return self._call_api( + 'cryptography.GetComponentsEncryptionHints', + self.crypto_stub.GetComponentsEncryptionHints, request, - PurlRequest, + ComponentsRequest, 'Sending data for encryption hints decoration (rqId: {rqId})...', + use_grpc=use_grpc, ) - def get_encryption_hints_in_range_for_purl(self, request: Dict) -> Optional[Dict]: + def get_encryption_hints_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ - Client function to call the rpc for GetHintsInRange for a list of purls + Client function to call the rpc for GetComponentsHintsInRange for a list of purls Args: - request (Dict): PurlRequest + request (Dict): ComponentsRequest + use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default) Returns: Optional[Dict]: HintsInRangeResponse, or None if the request was not successfull """ - return self._call_rpc( - self.crypto_stub.GetHintsInRange, + return self._call_api( + 'cryptography.GetComponentsHintsInRange', + self.crypto_stub.GetComponentsHintsInRange, request, - PurlRequest, + ComponentsRequest, 'Sending data for encryption hints in range decoration (rqId: {rqId})...', + use_grpc=use_grpc, ) - def get_versions_in_range_for_purl(self, request: Dict) -> Optional[Dict]: + def get_versions_in_range_for_purl(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ - Client function to call the rpc for GetVersionsInRange for a list of purls + Client function to call the rpc for GetComponentsVersionsInRange for a list of purls Args: - request (Dict): PurlRequest + request (Dict): ComponentsRequest + use_grpc (Optional[bool]): Whether to use gRPC or REST (None = use instance default) Returns: Optional[Dict]: VersionsInRangeResponse, or None if the request was not successfull """ - return self._call_rpc( - self.crypto_stub.GetVersionsInRange, + return self._call_api( + 'cryptography.GetComponentsVersionsInRange', + self.crypto_stub.GetComponentsVersionsInRange, request, - PurlRequest, + ComponentsRequest, 'Sending data for cryptographic versions in range decoration (rqId: {rqId})...', + use_grpc=use_grpc, ) - def get_licenses(self, request: Dict) -> Optional[Dict]: + def get_licenses(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ Client function to call the rpc for Licenses GetComponentsLicenses It will either use REST (default) or gRPC depending on the use_grpc flag @@ -718,28 +753,16 @@ def get_licenses(self, request: Dict) -> Optional[Dict]: Returns: Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull """ - if self.use_grpc: - return self._get_licenses_grpc(request) - else: - return self._get_licenses_rest(request) - - def _get_licenses_grpc(self, request: Dict) -> Optional[Dict]: - """ - Client function to call the rpc for GetComponentsLicenses - - Args: - request (Dict): ComponentsRequest - Returns: - Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull - """ - return self._call_rpc( + return self._call_api( + 'licenses.GetComponentsLicenses', self.license_stub.GetComponentsLicenses, request, ComponentsRequest, 'Sending data for license decoration (rqId: {rqId})...', + use_grpc=use_grpc, ) - def load_generic_headers(self, url: str = None): + def load_generic_headers(self, url: Optional[str] = None): """ Adds custom headers from req_headers to metadata. @@ -761,32 +784,91 @@ def load_generic_headers(self, url: str = None): # Start of REST Client Functions # - def rest_post(self, uri: str, request_id: str, data: dict) -> dict: + def _rest_get(self, uri: str, request_id: str, params: Optional[dict] = None) -> Optional[dict]: + """ + Send a GET request to the specified URI with optional query parameters. + + Args: + uri (str): URI to send GET request to + request_id (str): request id + params (dict, optional): Optional query parameters as dictionary + + Returns: + dict: JSON response or None + """ + if not uri: + self.print_stderr('Error: Missing URI. Cannot perform GET request.') + return None + self.print_trace(f'Sending REST GET request to {uri}...') + headers = self.headers.copy() + headers['x-request-id'] = request_id + retry = 0 + while retry <= self.retry_limit: + retry += 1 + try: + response = self.session.get(uri, headers=headers, params=params, timeout=self.timeout) + response.raise_for_status() # Raises an HTTPError for bad responses + return response.json() + except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e: + self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) sending GET request - {e}.') + raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e + except requests.exceptions.HTTPError as e: + self.print_stderr(f'ERROR: HTTP error sending GET request ({request_id}): {e}') + raise Exception( + f'ERROR: The SCANOSS API GET request failed with status {e.response.status_code} for {uri}' + ) from e + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: + if retry > self.retry_limit: # Timed out retry_limit or more times, fail + self.print_stderr(f'ERROR: {e.__class__.__name__} sending GET request ({request_id}): {e}') + raise Exception( + f'ERROR: The SCANOSS API GET request timed out ({e.__class__.__name__}) for {uri}' + ) from e + else: + self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...') + time.sleep(5) + except requests.exceptions.RequestException as e: + self.print_stderr(f'Error: Problem sending GET request to {uri}: {e}') + raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e + except Exception as e: + self.print_stderr( + f'ERROR: Exception ({e.__class__.__name__}) sending GET request ({request_id}) to {uri}: {e}' + ) + raise Exception(f'ERROR: The SCANOSS API GET request failed for {uri}') from e + return None + + def _rest_post(self, uri: str, request_id: str, data: dict) -> Optional[dict]: """ Post the specified data to the given URI. - :param request_id: request id - :param uri: URI to post to - :param data: json data to post - :return: JSON response or None + + Args: + uri (str): URI to post to + request_id (str): request id + data (dict): json data to post + + Returns: + dict: JSON response or None """ if not uri: self.print_stderr('Error: Missing URI. Cannot search for project.') return None self.print_trace(f'Sending REST POST data to {uri}...') - headers = self.headers - headers['x-request-id'] = request_id # send a unique request id for each post - retry = 0 # Add some retry logic to cater for timeouts, etc. + headers = self.headers.copy() + headers['x-request-id'] = request_id + retry = 0 while retry <= self.retry_limit: retry += 1 try: response = self.session.post(uri, headers=headers, json=data, timeout=self.timeout) response.raise_for_status() # Raises an HTTPError for bad responses return response.json() - except requests.exceptions.RequestException as e: - self.print_stderr(f'Error: Problem posting data to {uri}: {e}') except (requests.exceptions.SSLError, requests.exceptions.ProxyError) as e: self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.') raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e + except requests.exceptions.HTTPError as e: + self.print_stderr(f'ERROR: HTTP error POSTing data ({request_id}): {e}') + raise Exception( + f'ERROR: The SCANOSS Decoration API request failed with status {e.response.status_code} for {uri}' + ) from e except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: if retry > self.retry_limit: # Timed out retry_limit or more times, fail self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing decoration data ({request_id}): {e}') @@ -796,6 +878,9 @@ def rest_post(self, uri: str, request_id: str, data: dict) -> dict: else: self.print_stderr(f'Warning: {e.__class__.__name__} communicating with {self.url}. Retrying...') time.sleep(5) + except requests.exceptions.RequestException as e: + self.print_stderr(f'Error: Problem posting data to {uri}: {e}') + raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e except Exception as e: self.print_stderr( f'ERROR: Exception ({e.__class__.__name__}) POSTing data ({request_id}) to {uri}: {e}' @@ -803,75 +888,39 @@ def rest_post(self, uri: str, request_id: str, data: dict) -> dict: raise Exception(f'ERROR: The SCANOSS Decoration API request failed for {uri}') from e return None - def _get_licenses_rest(self, purls: Dict) -> Optional[Dict]: + def _call_rest(self, endpoint_key: str, request_input: dict, debug_msg: Optional[str] = None) -> Optional[Dict]: """ - Get the licenses for the given purls using REST API + Call a REST endpoint and return the response as a dictionary Args: - purls (Dict): Purl Request dictionary + endpoint_key (str): The key to lookup the REST endpoint in REST_ENDPOINTS + request_input (dict): The request data to send + debug_msg (str, optional): Debug message template that can include {rqId} placeholder. + Returns: - Optional[Dict]: ComponentsLicenseResponse, or None if the request was not successfull + dict: The parsed REST response as a dictionary, or None if something went wrong """ - if not purls: - self.print_stderr('ERROR: No message supplied to send to REST decoration service.') - return None - request_id = str(uuid.uuid4()) - self.print_debug(f'Sending data for Licenses via REST (request id: {request_id})...') - response = self.rest_post(f'{self.orig_url}{DEFAULT_URI_PREFIX}/licenses/components', request_id, purls) - self.print_trace(f'Received response for Licenses via REST (request id: {request_id}): {response}') - if response: - # Parse the JSON/Dict into the purl response - resp_obj = ParseDict(response, ComponentsLicenseResponse(), True) - if resp_obj: - self.print_debug(f'License Response: {resp_obj}') - if not self._check_status_response(resp_obj.status, request_id): - return None - del response['status'] - return response - return None + if endpoint_key not in REST_ENDPOINTS: + raise ScanossGrpcError(f'Unknown REST endpoint key: {endpoint_key}') - def _get_vulnerabilities_rest(self, purls: dict): - """ - Get the vulnerabilities for the given purls using REST API - :param purls: Purl Request dictionary - :return: Vulnerability Response, or None if the request was unsuccessful - """ - if not purls: - self.print_stderr('ERROR: No message supplied to send to REST decoration service.') - return None + endpoint_config = REST_ENDPOINTS[endpoint_key] + endpoint_path = endpoint_config['path'] + method = endpoint_config['method'] + endpoint_url = f'{self.orig_url}{DEFAULT_URI_PREFIX}{endpoint_path}' request_id = str(uuid.uuid4()) - self.print_debug(f'Sending data for Vulnerabilities via REST (request id: {request_id})...') - response = self.rest_post(f'{self.orig_url}{DEFAULT_URI_PREFIX}/vulnerabilities/components', request_id, purls) - self.print_trace(f'Received response for Vulnerabilities via REST (request id: {request_id}): {response}') - if response: - # Parse the JSON/Dict into the purl response - resp_obj = ParseDict(response, ComponentsVulnerabilityResponse(), True) - if resp_obj: - self.print_debug(f'Vulnerability Response: {resp_obj}') - if not self._check_status_response(resp_obj.status, request_id): - return None - del response['status'] - return response - return None - def _process_dep_file_rest(self, file, depth: int = 1) -> dict: - """ - Porcess a single dependency file using REST + if debug_msg: + self.print_debug(debug_msg.format(rqId=request_id)) - :param file: dependency file purls - :param depth: depth to search (default: 1) - :return: response JSON or None - """ - request_id = str(uuid.uuid4()) - self.print_debug(f'Sending data for Dependencies via REST (request id: {request_id})...') - file_request = {'files': [file], 'depth': depth} - response = self.rest_post( - f'{self.orig_url}{DEFAULT_URI_PREFIX}/dependencies/dependencies', request_id, file_request - ) - self.print_trace(f'Received response for Dependencies via REST (request id: {request_id}): {response}') - if response: - return response - return None + if method == 'GET': + response = self._rest_get(endpoint_url, request_id, params=request_input) + else: # POST + response = self._rest_post(endpoint_url, request_id, request_input) + + if response and 'status' in response and not self.check_status_response_rest(response['status'], request_id): + return None + + return response # From 01b528181836137dd524d5b4ad8c730bcbead7f0 Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:15:59 +0200 Subject: [PATCH 392/489] [SP-3346] feat: use gRPC by default instead of REST (#154) --- CHANGELOG.md | 4 ++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 9 ++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bb3fad7..904ebd61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.35.0] - 2025-10-07 +### Modified +- Use gRPC instead of REST for API calls + ## [1.34.0] - 2025-10-06 ### Added - Add REST API support for decoration commands diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 3339f1a3..805b3eea 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.34.0' +__version__ = '1.35.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index d524e705..b5c2a0b8 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1070,7 +1070,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_versions, c_licenses, ]: - p.add_argument('--grpc', action='store_true', help='Enable gRPC support') + p.add_argument('--grpc', action='store_true', default=True, help='Use gRPC (default)') + p.add_argument('--rest', action='store_true', dest='rest', help='Use REST instead of gRPC') # Help/Trace command options for p in [ @@ -1111,6 +1112,12 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') args = parser.parse_args() + + # TODO: Remove this hack once we go back to using REST as default + # Handle --rest overriding --grpc default + if hasattr(args, 'rest') and args.rest: + args.grpc = False + if args.version: ver(parser, args) sys.exit(0) From afc8cb62f158c39998750d2a6b413f23624d242d Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Mon, 13 Oct 2025 15:07:18 +0200 Subject: [PATCH 393/489] [SP-2991] feat: add recursive search support for folder hashing (#141) * [SP-2991] feat: add depth and min-cutoff-threshold arguments to folder hashing commands * [SP-2991] chore: update grpc definitions * [SP-2991] chore: update dockerfile, scanoss.json and setup.cfg * remove file extension filters to match go-minr criteria * [SP-3040] fix: remove unused code * [SP-2991]: chore: update changelog and bump version * fix hfh extension filter bug * [SP-2874]: rename to recursive_threshold * [SP-2874]: add min_accepted_score * [SP-2991] chore: update changelog, bump version --------- Co-authored-by: coresoftware dev --- CHANGELOG.md | 7 + src/scanoss/__init__.py | 2 +- .../api/common/v2/scanoss_common_pb2_grpc.py | 1 + src/scanoss/cli.py | 34 ++++ src/scanoss/constants.py | 5 +- src/scanoss/file_filters.py | 159 +----------------- src/scanoss/scanners/folder_hasher.py | 32 +++- src/scanoss/scanners/scanner_hfh.py | 28 ++- 8 files changed, 93 insertions(+), 175 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 904ebd61..a62ff75e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.36.0] - 2025-10-08 +### Added +- Add `--recursive-threshold` argument to folder scan command +- Add `--depth` argument to `folder-scan` and `folder-hash` commands + ## [1.35.0] - 2025-10-07 ### Modified - Use gRPC instead of REST for API calls @@ -677,3 +682,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.32.0]: https://github.com/scanoss/scanoss.py/compare/v1.31.5...v1.32.0 [1.33.0]: https://github.com/scanoss/scanoss.py/compare/v1.32.0...v1.33.0 [1.34.0]: https://github.com/scanoss/scanoss.py/compare/v1.33.0...v1.34.0 +[1.35.0]: https://github.com/scanoss/scanoss.py/compare/v1.34.0...v1.35.0 +[1.36.0]: https://github.com/scanoss/scanoss.py/compare/v1.35.0...v1.36.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 805b3eea..de19d8b4 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.35.0' +__version__ = '1.36.0' diff --git a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py index addf36f2..13118697 100644 --- a/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py +++ b/src/scanoss/api/common/v2/scanoss_common_pb2_grpc.py @@ -3,6 +3,7 @@ import warnings import grpc +import warnings GRPC_GENERATED_VERSION = '1.73.1' GRPC_VERSION = grpc.__version__ diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index b5c2a0b8..b4b3a793 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -59,7 +59,10 @@ from .components import Components from .constants import ( DEFAULT_API_TIMEOUT, + DEFAULT_HFH_DEPTH, + DEFAULT_HFH_MIN_ACCEPTED_SCORE, DEFAULT_HFH_RANK_THRESHOLD, + DEFAULT_HFH_RECURSIVE_THRESHOLD, DEFAULT_POST_SIZE, DEFAULT_RETRY, DEFAULT_TIMEOUT, @@ -869,6 +872,27 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Filter results to only show those with rank value at or below this threshold (e.g., --rank-threshold 3 ' 'returns results with rank 1, 2, or 3). Lower rank values indicate higher quality matches.', ) + p_folder_scan.add_argument( + '--depth', + type=int, + default=DEFAULT_HFH_DEPTH, + help=f'Defines how deep to scan the root directory (optional - default {DEFAULT_HFH_DEPTH})', + ) + p_folder_scan.add_argument( + '--recursive-threshold', + type=float, + default=DEFAULT_HFH_RECURSIVE_THRESHOLD, + help=f'Minimum score threshold to consider a match (optional - default: {DEFAULT_HFH_RECURSIVE_THRESHOLD})', + ) + p_folder_scan.add_argument( + '--min-accepted-score', + type=float, + default=DEFAULT_HFH_MIN_ACCEPTED_SCORE, + help=( + 'Only show results with a score at or above this threshold ' + f'(optional - default: {DEFAULT_HFH_MIN_ACCEPTED_SCORE})' + ), + ) p_folder_scan.set_defaults(func=folder_hashing_scan) # Sub-command: folder-hash @@ -887,6 +911,12 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 default='json', help='Result output format (optional - default: json)', ) + p_folder_hash.add_argument( + '--depth', + type=int, + default=DEFAULT_HFH_DEPTH, + help=f'Defines how deep to hash the root directory (optional - default {DEFAULT_HFH_DEPTH})', + ) p_folder_hash.set_defaults(func=folder_hash) # Output options @@ -2456,6 +2486,9 @@ def folder_hashing_scan(parser, args): client=client, scanoss_settings=scanoss_settings, rank_threshold=args.rank_threshold, + depth=args.depth, + recursive_threshold=args.recursive_threshold, + min_accepted_score=args.min_accepted_score, ) if scanner.scan(): @@ -2489,6 +2522,7 @@ def folder_hash(parser, args): scan_dir=args.scan_dir, config=folder_hasher_config, scanoss_settings=scanoss_settings, + depth=args.depth, ) folder_hasher.hash_directory(args.scan_dir) diff --git a/src/scanoss/constants.py b/src/scanoss/constants.py index 92fc15b7..989f2008 100644 --- a/src/scanoss/constants.py +++ b/src/scanoss/constants.py @@ -13,4 +13,7 @@ DEFAULT_API_TIMEOUT = 600 -DEFAULT_HFH_RANK_THRESHOLD = 5 \ No newline at end of file +DEFAULT_HFH_RANK_THRESHOLD = 5 +DEFAULT_HFH_DEPTH = 1 +DEFAULT_HFH_RECURSIVE_THRESHOLD = 0.8 +DEFAULT_HFH_MIN_ACCEPTED_SCORE = 0.15 diff --git a/src/scanoss/file_filters.py b/src/scanoss/file_filters.py index cb8298a8..33595374 100644 --- a/src/scanoss/file_filters.py +++ b/src/scanoss/file_filters.py @@ -269,162 +269,6 @@ 'sqlite3', } -# TODO: For hfh add the .gitignore patterns -DEFAULT_SKIPPED_EXT_HFH = { - '.1', - '.2', - '.3', - '.4', - '.5', - '.6', - '.7', - '.8', - '.9', - '.ac', - '.adoc', - '.am', - '.asciidoc', - '.bmp', - '.build', - '.cfg', - '.chm', - '.class', - '.cmake', - '.cnf', - '.conf', - '.config', - '.contributors', - '.copying', - '.crt', - '.csproj', - '.css', - '.csv', - '.dat', - '.data', - '.dtd', - '.dts', - '.iws', - '.c9', - '.c9revisions', - '.dtsi', - '.dump', - '.eot', - '.eps', - '.geojson', - '.gif', - '.glif', - '.gmo', - '.guess', - '.hex', - '.htm', - '.html', - '.ico', - '.iml', - '.in', - '.inc', - '.info', - '.ini', - '.ipynb', - '.jpeg', - '.jpg', - '.json', - '.jsonld', - '.lock', - '.log', - '.m4', - '.map', - '.md5', - '.meta', - '.mk', - '.mxml', - '.o', - '.otf', - '.out', - '.pbtxt', - '.pdf', - '.pem', - '.phtml', - '.plist', - '.png', - '.prefs', - '.properties', - '.pyc', - '.qdoc', - '.result', - '.rgb', - '.rst', - '.scss', - '.sha', - '.sha1', - '.sha2', - '.sha256', - '.sln', - '.spec', - '.sub', - '.svg', - '.svn-base', - '.tab', - '.template', - '.test', - '.tex', - '.tiff', - '.ttf', - '.txt', - '.utf-8', - '.vim', - '.wav', - '.woff', - '.woff2', - '.xht', - '.xhtml', - '.xml', - '.xpm', - '.xsd', - '.xul', - '.yaml', - '.yml', - '.wfp', - '.editorconfig', - '.dotcover', - '.pid', - '.lcov', - '.egg', - '.manifest', - '.cache', - '.coverage', - '.cover', - '.gem', - '.lst', - '.pickle', - '.pdb', - '.gml', - '.pot', - '.plt', - '.whml', - '.pom', - '.smtml', - '.min.js', - '.mf', - '.base64', - '.s', - '.diff', - '.patch', - '.rules', - # File endings - '-doc', - 'config', - 'news', - 'readme', - 'swiftdoc', - 'texidoc', - 'todo', - 'version', - 'ignore', - 'manifest', - 'sqlite', - 'sqlite3', -} - class FileFilters(ScanossBase): """ @@ -707,9 +551,8 @@ def _should_skip_file(self, file_rel_path: str) -> bool: # noqa: PLR0911 bool: True if file should be skipped, False otherwise """ file_name = os.path.basename(file_rel_path) - + DEFAULT_SKIPPED_EXT_LIST = {} if self.is_folder_hashing_scan else DEFAULT_SKIPPED_EXT DEFAULT_SKIPPED_FILES_LIST = DEFAULT_SKIPPED_FILES_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_FILES - DEFAULT_SKIPPED_EXT_LIST = DEFAULT_SKIPPED_EXT_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_EXT if not self.hidden_files_folders and file_name.startswith('.'): self.print_debug(f'Skipping file: {file_rel_path} (hidden file)') diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index ad1bad32..2e516780 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -6,6 +6,7 @@ from progress.bar import Bar +from scanoss.constants import DEFAULT_HFH_DEPTH from scanoss.file_filters import FileFilters from scanoss.scanoss_settings import ScanossSettings from scanoss.scanossbase import ScanossBase @@ -15,8 +16,6 @@ MINIMUM_FILE_COUNT = 8 MINIMUM_CONCATENATED_NAME_LENGTH = 32 -MAXIMUM_FILE_NAME_LENGTH = 32 - class DirectoryNode: """ @@ -72,6 +71,12 @@ class FolderHasher: It builds a directory tree (DirectoryNode) and computes the associated hash data for the folder. + + Args: + scan_dir (str): The directory to be hashed. + config (FolderHasherConfig): Configuration parameters for the folder hasher. + scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss. + depth (int): How many levels to hash from the root directory (default: 1). """ def __init__( @@ -79,6 +84,7 @@ def __init__( scan_dir: str, config: FolderHasherConfig, scanoss_settings: Optional[ScanossSettings] = None, + depth: int = DEFAULT_HFH_DEPTH, ): self.base = ScanossBase( debug=config.debug, @@ -101,6 +107,7 @@ def __init__( self.scan_dir = scan_dir self.tree = None + self.depth = depth def hash_directory(self, path: str) -> dict: """ @@ -123,7 +130,10 @@ def hash_directory(self, path: str) -> dict: return tree - def _build_root_node(self, path: str) -> DirectoryNode: + def _build_root_node( + self, + path: str, + ) -> DirectoryNode: """ Build a directory tree from the given path with file information. @@ -140,7 +150,7 @@ def _build_root_node(self, path: str) -> DirectoryNode: root_node = DirectoryNode(str(root)) all_files = [ - f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) <= MAXIMUM_FILE_NAME_LENGTH + f for f in root.rglob('*') if f.is_file() ] filtered_files = self.file_filters.get_filtered_files_from_files(all_files, str(root)) @@ -180,7 +190,7 @@ def _build_root_node(self, path: str) -> DirectoryNode: bar.finish() return root_node - def _hash_calc_from_node(self, node: DirectoryNode) -> dict: + def _hash_calc_from_node(self, node: DirectoryNode, current_depth: int = 1) -> dict: """ Recursively compute folder hash data for a directory node. @@ -189,12 +199,13 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: Args: node (DirectoryNode): The directory node to compute the hash for. + current_depth (int): The current depth level (1-based, root is depth 1). Returns: dict: The computed hash data for the node. """ hash_data = self._hash_calc(node) - + # Safely calculate relative path try: node_path = Path(node.path).resolve() @@ -204,13 +215,18 @@ def _hash_calc_from_node(self, node: DirectoryNode) -> dict: # If relative_to fails, use the node path as is or a fallback rel_path = Path(node.path).name if node.path else Path('.') + # Only process children if we haven't reached the depth limit + children = [] + if current_depth < self.depth: + children = [self._hash_calc_from_node(child, current_depth + 1) for child in node.children.values()] + return { 'path_id': str(rel_path), 'sim_hash_names': f'{hash_data["name_hash"]:02x}' if hash_data['name_hash'] is not None else None, 'sim_hash_content': f'{hash_data["content_hash"]:02x}' if hash_data['content_hash'] is not None else None, 'sim_hash_dir_names': f'{hash_data["dir_hash"]:02x}' if hash_data['dir_hash'] is not None else None, 'lang_extensions': hash_data['lang_extensions'], - 'children': [self._hash_calc_from_node(child) for child in node.children.values()], + 'children': children, } def _hash_calc(self, node: DirectoryNode) -> dict: @@ -237,8 +253,6 @@ def _hash_calc(self, node: DirectoryNode) -> dict: for file in node.files: key_str = file.key_str - if key_str in processed_hashes: - continue file_name = os.path.basename(file.path) diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 9f4df38c..2418d4db 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -29,7 +29,12 @@ from progress.spinner import Spinner -from scanoss.constants import DEFAULT_HFH_RANK_THRESHOLD +from scanoss.constants import ( + DEFAULT_HFH_DEPTH, + DEFAULT_HFH_MIN_ACCEPTED_SCORE, + DEFAULT_HFH_RANK_THRESHOLD, + DEFAULT_HFH_RECURSIVE_THRESHOLD, +) from scanoss.cyclonedx import CycloneDx from scanoss.file_filters import FileFilters from scanoss.scanners.folder_hasher import FolderHasher @@ -48,13 +53,16 @@ class ScannerHFH: and calculates simhash values based on file names and content to detect folder-level similarities. """ - def __init__( + def __init__( # noqa: PLR0913 self, scan_dir: str, config: ScannerConfig, client: Optional[ScanossGrpc] = None, scanoss_settings: Optional[ScanossSettings] = None, rank_threshold: int = DEFAULT_HFH_RANK_THRESHOLD, + depth: int = DEFAULT_HFH_DEPTH, + recursive_threshold: float = DEFAULT_HFH_RECURSIVE_THRESHOLD, + min_accepted_score: float = DEFAULT_HFH_MIN_ACCEPTED_SCORE, ): """ Initialize the ScannerHFH. @@ -65,6 +73,9 @@ def __init__( client (ScanossGrpc): gRPC client for communicating with the scanning service. scanoss_settings (Optional[ScanossSettings]): Optional settings for Scanoss. rank_threshold (int): Get results with rank below this threshold (default: 5). + depth (int): How many levels to scan (default: 1). + recursive_threshold (float): Minimum score threshold to consider a match (default: 0.25). + min_accepted_score (float): Only show results with a score at or above this threshold (default: 0.15). """ self.base = ScanossBase( debug=config.debug, @@ -87,12 +98,15 @@ def __init__( scan_dir=scan_dir, config=config, scanoss_settings=scanoss_settings, + depth=depth, ) self.scan_dir = scan_dir self.client = client self.scan_results = None self.rank_threshold = rank_threshold + self.recursive_threshold = recursive_threshold + self.min_accepted_score = min_accepted_score def scan(self) -> Optional[Dict]: """ @@ -102,8 +116,10 @@ def scan(self) -> Optional[Dict]: Optional[Dict]: The folder hash response from the gRPC client, or None if an error occurs. """ hfh_request = { - 'root': self.folder_hasher.hash_directory(self.scan_dir), + 'root': self.folder_hasher.hash_directory(path=self.scan_dir), 'rank_threshold': self.rank_threshold, + 'recursive_threshold': self.recursive_threshold, + 'min_accepted_score': self.min_accepted_score, } spinner = Spinner('Scanning folder...') @@ -193,7 +209,7 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 } ] } - + get_vulnerabilities_json_request = { 'purls': [{'purl': purl, 'requirement': best_match_version['version']}], } @@ -210,10 +226,10 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 error_msg = 'ERROR: Failed to produce CycloneDX output' self.base.print_stderr(error_msg) return None - + if vulnerabilities: cdx_output = cdx.append_vulnerabilities(cdx_output, vulnerabilities, purl) - + return json.dumps(cdx_output, indent=2) except Exception as e: self.base.print_stderr(f'ERROR: Failed to get license information: {e}') From ff6b8f7ac7668f4be8e5e3966b8c4d1cf51eda98 Mon Sep 17 00:00:00 2001 From: Alex-1089 Date: Fri, 17 Oct 2025 15:35:56 +0100 Subject: [PATCH 394/489] SP-3354_delta-scan-setup * Added delta command with copy subcommand * Intended to be used by delta scans via integration Co-authored-by: eeisegn --- CHANGELOG.md | 5 + src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 70 +++++++++++++- src/scanoss/delta.py | 197 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 src/scanoss/delta.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a62ff75e..bdd41dc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.37.0] - 2025-10-17 +### Added +- Added delta folder and file copy command + ## [1.36.0] - 2025-10-08 ### Added - Add `--recursive-threshold` argument to folder scan command @@ -684,3 +688,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.34.0]: https://github.com/scanoss/scanoss.py/compare/v1.33.0...v1.34.0 [1.35.0]: https://github.com/scanoss/scanoss.py/compare/v1.34.0...v1.35.0 [1.36.0]: https://github.com/scanoss/scanoss.py/compare/v1.35.0...v1.36.0 +[1.37.0]: https://github.com/scanoss/scanoss.py/compare/v1.36.0...v1.37.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index de19d8b4..802f07c5 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.36.0' +__version__ = '1.37.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index b4b3a793..044efc84 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -33,6 +33,7 @@ import pypac from scanoss.cryptography import Cryptography, create_cryptography_config_from_args +from scanoss.delta import Delta from scanoss.export.dependency_track import DependencyTrackExporter from scanoss.inspection.dependency_track.project_violation import ( DependencyTrackProjectViolationPolicyCheck, @@ -919,6 +920,33 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) p_folder_hash.set_defaults(func=folder_hash) + # Sub-command: delta + p_delta = subparsers.add_parser( + 'delta', + aliases=['dl'], + description=f'SCANOSS Delta commands: {__version__}', + help='Delta support commands', + ) + + delta_sub = p_delta.add_subparsers( + title='Delta Commands', + dest='subparsercmd', + description='Delta sub-commands', + help='Delta sub-commands' + ) + + # Delta Sub-command: copy + p_copy = delta_sub.add_parser( + 'copy', + aliases=['cp'], + description=f'Copy file list into delta dir: {__version__}', + help='Copy the given list of files into a delta directory', + ) + p_copy.add_argument('--input', '-i', type=str, required=True, help='Input file with diff list') + p_copy.add_argument('--folder', '-fd', type=str, help='Delta folder to copy into') + p_copy.add_argument('--root', '-rd', type=str, help='Root directory to place delta folder') + p_copy.set_defaults(func=delta_copy) + # Output options for p in [ p_scan, @@ -939,6 +967,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_hints, p_crypto_versions_in_range, c_licenses, + p_copy, ]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -1136,6 +1165,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_versions_in_range, c_licenses, e_dt, + p_copy ]: p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') @@ -1156,7 +1186,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 sys.exit(1) elif ( args.subparser - in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins', 'crypto', 'cr', 'export', 'exp') + in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins', + 'crypto', 'cr', 'export', 'exp', 'delta', 'dl') ) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed sys.exit(1) @@ -2603,6 +2634,43 @@ def initialise_empty_file(filename: str): print_stderr(f'Error: Unable to create output file {filename}: {e}') sys.exit(1) +def delta_copy(parser, args): + """ + Handle delta copy command. + + Copies files listed in an input file to a target directory while preserving + their directory structure. Creates a unique delta directory if none is specified. + + Parameters + ---------- + parser : ArgumentParser + Command line parser object for help display + args : Namespace + Parsed command line arguments containing: + - input: Path to file containing list of files to copy + - folder: Optional target directory path + - output: Optional output file path + """ + # Validate required input file parameter + if args.input is None: + print_stderr('ERROR: Input file is required for copying') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + # Initialise output file if specified + if args.output: + initialise_empty_file(args.output) + try: + # Create and configure delta copy command + delta = Delta(debug=args.debug, trace=args.trace, quiet=args.quiet, filepath=args.input, folder=args.folder, + output=args.output, root_dir=args.root) + # Execute copy and exit with appropriate status code + status, _ = delta.copy() + sys.exit(status) + except Exception as e: + print_stderr(e) + if args.debug: + traceback.print_exc() + sys.exit(1) def main(): """ diff --git a/src/scanoss/delta.py b/src/scanoss/delta.py new file mode 100644 index 00000000..d3de7b07 --- /dev/null +++ b/src/scanoss/delta.py @@ -0,0 +1,197 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import os +import shutil +import tempfile +from typing import Optional + +from .scanossbase import ScanossBase + + +class Delta(ScanossBase): + """ + Handle delta scan operations by copying files into a dedicated delta directory. + + This class manages the creation of delta directories and copying of specified files + while preserving the directory structure. Files are read from an input file where each + line contains a file path to copy. + """ + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + filepath: str = None, + folder: str = None, + output: str = None, + root_dir: str = None, + ): + """ + Initialise the Delta instance. + + :param debug: Enable debug logging. + :param trace: Enable trace logging. + :param quiet: Enable quiet mode (suppress non-essential output). + :param filepath: Path to an input file containing a list of files to copy. + :param folder: A target delta directory path (auto-generated if not provided). + :param output: Output file path for the delta directory location (stdout if not provided). + """ + super().__init__(debug, trace, quiet) + self.filepath = filepath + self.folder = folder + self.output = output + self.root_dir = root_dir if root_dir else '.' + + def copy(self, input_file: str = None): + """ + Copy files listed in the input file to the delta directory. + + Reads the input file line by line, where each line contains a file path. + Creates the delta directory if it doesn't exist, then copies each file + while preserving its directory structure. + + :return: Tuple of (status_code, folder_path) where status_code is 0 for success, + 1 for error, and folder_path is the delta directory path + """ + input_file = input_file if input_file else self.filepath + if not input_file: + self.print_stderr('ERROR: No input file specified') + return 1, '' + # Validate that an input file exists + if not os.path.isfile(input_file): + self.print_stderr(f'ERROR: Input file {input_file} does not exist or is not a file') + return 1, '' + # Load the input file and validate it contains valid file paths + files = self.load_input_file(input_file) + if files is None: + return 1, '' + # Create delta dir (folder) + delta_folder = self.create_delta_dir(self.folder, self.root_dir) + if not delta_folder: + return 1, '' + # Print delta folder location to output + self.print_to_file_or_stdout(delta_folder, self.output) + # Process each file and copy it to the delta dir + for source_file in files: + # Normalise the source path to handle ".." and redundant separators + normalised_source = os.path.normpath(source_file) + if '..' in normalised_source: + self.print_stderr(f'WARNING: Source path escapes root directory for {source_file}. Skipping.') + continue + # Resolve to the absolute path for source validation + abs_source = os.path.abspath(os.path.join(self.root_dir, normalised_source)) + # Check if the source file exists and is a file + if not os.path.exists(abs_source) or not os.path.isfile(abs_source): + self.print_stderr(f'WARNING: File {source_file} does not exist or is not a file, skipping') + continue + # Use a normalised source for destination to prevent traversal + dest_path = os.path.normpath(os.path.join(self.root_dir, delta_folder, normalised_source.lstrip(os.sep))) + # Final safety check: ensure destination is within the delta folder + abs_dest = os.path.abspath(dest_path) + abs_folder = os.path.abspath(os.path.join(self.root_dir, delta_folder)) + if not abs_dest.startswith(abs_folder + os.sep): + self.print_stderr( + f'WARNING: Destination path ({abs_dest}) escapes delta directory for {source_file}. Skipping.') + continue + # Create the destination directory if it doesn't exist and copy the file + try: + dest_dir = os.path.dirname(dest_path) + if dest_dir: + self.print_trace(f'Creating directory {dest_dir}...') + os.makedirs(dest_dir, exist_ok=True) + self.print_debug(f'Copying {source_file} to {dest_path} ...') + shutil.copy(abs_source, dest_path) + except (OSError, shutil.Error) as e: + self.print_stderr(f'ERROR: Failed to copy {source_file} to {dest_path}: {e}') + return 1, '' + return 0, delta_folder + + def create_delta_dir(self, folder: str, root_dir: str = '.') -> str or None: + """ + Create the delta directory. + + If no folder is specified, creates a unique temporary directory with + a 'delta-' prefix in the current directory. If a folder is specified, + validates that it doesn't already exist before creating it. + + :param root_dir: Root directory to create the delta directory in (default: current directory) + :param folder: Optional target directory + :return: Path to the delta directory, or None if it already exists or creation fails + """ + if folder: + # Resolve a relative folder under root_dir so checks/creation apply to the right place + resolved = folder if os.path.isabs(folder) else os.path.join(root_dir, folder) + resolved = os.path.normpath(resolved) + # Validate the target directory doesn't already exist and create it + if os.path.exists(resolved): + self.print_stderr(f'ERROR: Folder {resolved} already exists.') + return None + else: + try: + self.print_debug(f'Creating delta directory {resolved}...') + os.makedirs(resolved) + except (OSError, IOError) as e: + self.print_stderr(f'ERROR: Failed to create directory {resolved}: {e}') + return None + else: + # Create a unique temporary directory in the given root directory + try: + self.print_debug(f'Creating temporary delta directory in {root_dir} ...') + folder = tempfile.mkdtemp(prefix="delta-", dir=root_dir) + if folder: + folder = os.path.relpath(folder, start=root_dir) # Get the relative path from root_dir + self.print_debug(f'Created temporary delta directory: {folder}') + except (OSError, IOError) as e: + self.print_stderr(f'ERROR: Failed to create temporary directory in {root_dir}: {e}') + return None + return folder + + def load_input_file(self, input_file: str) -> Optional[list[str]]: + """ + Loads and parses the input file line by line. Each line in the input + file represents a source file path, which will be stripped of trailing + whitespace and appended to the resulting list if it is not empty. + + :param input_file: The path to the input file to be read. + :type input_file: String + :return: A list of source file paths extracted from the input file, + or None if an error occurs or the file path is invalid. + :rtype: An array list[str] or None + """ + files = [] + if input_file: + try: + with open(input_file, 'r', encoding='utf-8') as f: + for line in f: + source_file = line.rstrip() + if source_file: + # Save the file path without any leading separators + files.append(source_file.lstrip(os.sep)) + # End of for loop + except (OSError, IOError) as e: + self.print_stderr(f'ERROR: Failed to read input file; {input_file}: {e}') + return None + self.print_debug(f'Loaded {len(files)} files from input file.') + return files From 099ede5b7b8aea458f9fd894e5b1f83c52785cd7 Mon Sep 17 00:00:00 2001 From: Alex-1089 Date: Tue, 21 Oct 2025 13:48:26 +0100 Subject: [PATCH 395/489] SP-3561_conversion-issues * Fixed source filtering bug for spdx conversion * Added source filtering for cyclonedx conversion --- CHANGELOG.md | 7 +++++++ src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 9 +++++++-- src/scanoss/spdxlite.py | 6 ++++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd41dc2..46079c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.37.1] - 2025-10-21 +### Added +- Added source filtering to cyclonedx conversion +### Fixed +- Fixed dependencies being skipped during spdx conversion + ## [1.37.0] - 2025-10-17 ### Added - Added delta folder and file copy command @@ -689,3 +695,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.35.0]: https://github.com/scanoss/scanoss.py/compare/v1.34.0...v1.35.0 [1.36.0]: https://github.com/scanoss/scanoss.py/compare/v1.35.0...v1.36.0 [1.37.0]: https://github.com/scanoss/scanoss.py/compare/v1.36.0...v1.37.0 +[1.37.1]: https://github.com/scanoss/scanoss.py/compare/v1.37.0...v1.37.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 802f07c5..78e1e62f 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.37.0' +__version__ = '1.37.1' diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 555ba4ad..e1012605 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -152,7 +152,11 @@ def parse(self, data: dict): # noqa: PLR0912, PLR0915 fdl = [] if licenses: for lic in licenses: - fdl.append({'id': lic.get('name')}) + name = lic.get('name') + source = lic.get('source') + if source not in ('component_declared', 'license_file', 'file_header'): + continue + fdl.append({'id': name}) fd['licenses'] = fdl cdx[purl] = fd # self.print_stderr(f'VD: {vdx}') @@ -295,7 +299,8 @@ def produce_from_str(self, json_str: str, output_file: str = None) -> bool: except Exception as e: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return False - return self.produce_from_json(data, output_file) + success, _ = self.produce_from_json(data, output_file) + return success def _normalize_vulnerability_id(self, vuln: dict) -> tuple[str, str]: """ diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 7313b271..3e13af89 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -226,7 +226,9 @@ def _process_licenses(self, licenses: list) -> list: Process license information and remove duplicates. This method filters license information to include only licenses from trusted sources - ('component_declared' or 'license_file') and removes any duplicate license names. + ('component_declared', 'license_file', 'file_header'). Licenses with an unspecified + source (None or '') are allowed. Non-empty, non-allowed sources are excluded. It also + removes any duplicate license names. The result is a simplified list of license dictionaries containing only the 'id' field. Args: @@ -247,7 +249,7 @@ def _process_licenses(self, licenses: list) -> list: for license_info in licenses: name = license_info.get('name') source = license_info.get('source') - if source not in ("component_declared", "license_file", "file_header"): + if source not in (None, '') and source not in ("component_declared", "license_file", "file_header"): continue if name and name not in seen_names: processed_licenses.append({'id': name}) From a01b51e8031cdd3b45d9a9806691a295df77e658 Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Fri, 24 Oct 2025 15:18:21 +0200 Subject: [PATCH 396/489] [SP-3572] feat: add support for SCANOSS_DEBUG env variable (#158) --- CHANGELOG.md | 5 +++++ CLIENT_HELP.md | 22 ++++++++++++++++++++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 45 ++++++++++++++++++++++++++++++++--------- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46079c43..47e58f01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.38.0] - 2025-10-24 +### Added +- Add support for settings debug mode via `SCANOSS_DEBUG` environment variable + ## [1.37.1] - 2025-10-21 ### Added - Added source filtering to cyclonedx conversion @@ -696,3 +700,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.36.0]: https://github.com/scanoss/scanoss.py/compare/v1.35.0...v1.36.0 [1.37.0]: https://github.com/scanoss/scanoss.py/compare/v1.36.0...v1.37.0 [1.37.1]: https://github.com/scanoss/scanoss.py/compare/v1.37.0...v1.37.1 +[1.38.0]: https://github.com/scanoss/scanoss.py/compare/v1.37.1...v1.38.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 5707adbc..074e3b20 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -130,6 +130,28 @@ pip3 install --upgrade https://github.com/pietrodn/grpcio-mac-arm-build/releases This command above will install `grpcio` `1.5.1` for Python `3.9`. To install for `3.10` simply replace the `cp39` with `cp310`. +## Debug Mode +The SCANOSS CLI supports debug mode to provide additional diagnostic information during command execution. This is useful for troubleshooting issues or understanding the internal operations of the CLI. + +There are two ways to enable debug mode: + +### Debug Mode via Environment Variable +Set the `SCANOSS_DEBUG` environment variable to `true`: +```bash +export SCANOSS_DEBUG=true +scanoss-py scan -o results.json src +``` + +This method is particularly useful when you want debug output enabled for multiple consecutive commands without having to add the flag each time. + +### Debug Mode via Command Line Flag +Use the `-d` or `--debug` flag with any command: +```bash +scanoss-py scan -d -o results.json src +``` + +**Note:** The command line flag will override the environment variable setting if both are present. + ## Command Execution There are multiple commands (and sub commands) available through `scanoss-py`. Detailed help is available for all directly from the CLI itself: diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 78e1e62f..23ddbcbd 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.37.1' +__version__ = '1.38.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 044efc84..b4984184 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -929,10 +929,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) delta_sub = p_delta.add_subparsers( - title='Delta Commands', - dest='subparsercmd', - description='Delta sub-commands', - help='Delta sub-commands' + title='Delta Commands', dest='subparsercmd', description='Delta sub-commands', help='Delta sub-commands' ) # Delta Sub-command: copy @@ -1165,9 +1162,15 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_versions_in_range, c_licenses, e_dt, - p_copy + p_copy, ]: - p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages') + p.add_argument( + '--debug', + '-d', + action='store_true', + default=os.environ.get('SCANOSS_DEBUG', '').lower() == 'true', + help='Enable debug messages (can also be set via environment variable SCANOSS_DEBUG)', + ) p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts') p.add_argument('--quiet', '-q', action='store_true', help='Enable quiet mode') @@ -1186,8 +1189,21 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 sys.exit(1) elif ( args.subparser - in ('utils', 'ut', 'component', 'comp', 'inspect', 'insp', 'ins', - 'crypto', 'cr', 'export', 'exp', 'delta', 'dl') + in ( + 'utils', + 'ut', + 'component', + 'comp', + 'inspect', + 'insp', + 'ins', + 'crypto', + 'cr', + 'export', + 'exp', + 'delta', + 'dl', + ) ) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed sys.exit(1) @@ -2634,6 +2650,7 @@ def initialise_empty_file(filename: str): print_stderr(f'Error: Unable to create output file {filename}: {e}') sys.exit(1) + def delta_copy(parser, args): """ Handle delta copy command. @@ -2661,8 +2678,15 @@ def delta_copy(parser, args): initialise_empty_file(args.output) try: # Create and configure delta copy command - delta = Delta(debug=args.debug, trace=args.trace, quiet=args.quiet, filepath=args.input, folder=args.folder, - output=args.output, root_dir=args.root) + delta = Delta( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + filepath=args.input, + folder=args.folder, + output=args.output, + root_dir=args.root, + ) # Execute copy and exit with appropriate status code status, _ = delta.copy() sys.exit(status) @@ -2672,6 +2696,7 @@ def delta_copy(parser, args): traceback.print_exc() sys.exit(1) + def main(): """ Run the ScanOSS CLI From 5650419dd1026ea182cf78c08d6bcaea0b40f20c Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Mon, 27 Oct 2025 11:38:17 -0300 Subject: [PATCH 397/489] Feat/added code quality format to convert subcomand and added new match summary into inspect subcommand * feat:SP-3567 adds GitLab Code Quality report format support * feat: SP-3574 add GitLab matches summary report generator into inspect command Implements GitLab-compatible Markdown summary reports for SCANOSS matches with the following changes: - Add new 'inspect gitlab matches' CLI command to generate match summaries - Create MatchSummary class to process and format scan results with collapsible sections - Extract table generation utilities into shared markdown_utils module - Extract JSON file loading into shared file_utils module - Add support for GitLab file links with line range anchors * chore:SP-3582 Adds documentation for the new GitLab Code Quality format and GitLab matches summary features in CLIENT_HELP.md * chore:SP-3589 Enhanced the match summary with clickable links and improved formatting --------- Co-authored-by: eeisegn --- CHANGELOG.md | 10 + CLIENT_HELP.md | 14 +- src/scanoss/cli.py | 141 ++++++++- src/scanoss/gitlabqualityreport.py | 185 +++++++++++ .../dependency_track/project_violation.py | 5 +- src/scanoss/inspection/policy_check.py | 42 --- src/scanoss/inspection/raw/__init__.py | 0 .../inspection/raw/component_summary.py | 55 +++- src/scanoss/inspection/raw/copyleft.py | 5 +- src/scanoss/inspection/raw/license_summary.py | 50 ++- src/scanoss/inspection/raw/match_summary.py | 290 ++++++++++++++++++ src/scanoss/inspection/raw/raw_base.py | 15 +- .../inspection/raw/undeclared_component.py | 5 +- src/scanoss/inspection/utils/file_utils.py | 44 +++ .../inspection/utils/markdown_utils.py | 63 ++++ src/scanoss/scanners/folder_hasher.py | 1 + .../utils/scanoss_scan_results_utils.py | 41 +++ 17 files changed, 902 insertions(+), 64 deletions(-) create mode 100644 src/scanoss/gitlabqualityreport.py create mode 100644 src/scanoss/inspection/raw/__init__.py create mode 100644 src/scanoss/inspection/raw/match_summary.py create mode 100644 src/scanoss/inspection/utils/file_utils.py create mode 100644 src/scanoss/inspection/utils/markdown_utils.py create mode 100644 src/scanoss/utils/scanoss_scan_results_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 47e58f01..aef5fbb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.39.0] - 2025-10-24 +### Added +- Added `glc-codequality` format to convert subcommand +- Added `inspect gitlab matches` subcommand to generate GitLab-compatible Markdown match summary from SCANOSS scan results +- Added utility modules for shared functionality (`markdown_utils.py` and `file_utils.py`) +### Changed +- Refactored table generation utilities into shared `markdown_utils` module +- Refactored JSON file loading into shared `file_utils` module + ## [1.38.0] - 2025-10-24 ### Added - Add support for settings debug mode via `SCANOSS_DEBUG` environment variable @@ -701,3 +710,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.37.0]: https://github.com/scanoss/scanoss.py/compare/v1.36.0...v1.37.0 [1.37.1]: https://github.com/scanoss/scanoss.py/compare/v1.37.0...v1.37.1 [1.38.0]: https://github.com/scanoss/scanoss.py/compare/v1.37.1...v1.38.0 +[1.39.0]: https://github.com/scanoss/scanoss.py/compare/v1.38.0...v1.39.0 \ No newline at end of file diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 074e3b20..877c6b8a 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -260,7 +260,7 @@ scanoss-py scan src -hdr "x-api-key:12345" -hdr "Authorization: Bearer --dt-url --dt-projectname --dt-projectversion --dt-apikey --format md --output project-violations.md ``` +#### Inspect GitLab Component Match Summary Markdown Output +The following command can be used to generate a component match summary in Markdown format for GitLab: +```bash +scanoss-py inspect gitlab matches --input -lpr --output gitlab-component-match-summary.md +``` ### Folder-Scan a Project Folder diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index b4984184..0542ebeb 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -40,6 +40,7 @@ ) from scanoss.inspection.raw.component_summary import ComponentSummary from scanoss.inspection.raw.license_summary import LicenseSummary +from scanoss.inspection.raw.match_summary import MatchSummary from scanoss.scanners.container_scanner import ( DEFAULT_SYFT_COMMAND, DEFAULT_SYFT_TIMEOUT, @@ -73,6 +74,7 @@ from .csvoutput import CsvOutput from .cyclonedx import CycloneDx from .filecount import FileCount +from .gitlabqualityreport import GitLabQualityReport from .inspection.raw.copyleft import Copyleft from .inspection.raw.undeclared_component import UndeclaredComponent from .results import Results @@ -283,7 +285,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--format', '-f', type=str, - choices=['cyclonedx', 'spdxlite', 'csv'], + choices=['cyclonedx', 'spdxlite', 'csv', 'glc-codequality'], default='spdxlite', help='Output format (optional - default: spdxlite)', ) @@ -794,6 +796,66 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Timeout (in seconds) for API communication (optional - default 300 sec)', ) + + # ============================================================================== + # GitLab Integration Parser + # ============================================================================== + # Main parser for GitLab-specific inspection commands and report generation + p_gitlab_sub = p_inspect_sub.add_parser( + 'gitlab', + aliases=['glc'], + description='Generate GitLab-compatible reports from SCANOSS scan results (Markdown summaries)', + help='Generate GitLab integration reports', + ) + + # GitLab sub-commands parser + # Provides access to different GitLab report formats and inspection tools + p_gitlab_sub_parser = p_gitlab_sub.add_subparsers( + title='GitLab Report Types', + dest='subparser_subcmd', + description='Available GitLab report formats for scan result analysis', + help='Select the type of GitLab report to generate', + ) + + # ============================================================================== + # GitLab Matches Summary Command + # ============================================================================== + # Analyzes scan results and generates a GitLab-compatible Markdown summary + p_gl_inspect_matches = p_gitlab_sub_parser.add_parser( + 'matches', + aliases=['ms'], + description='Generate a Markdown summary report of scan matches for GitLab integration', + help='Generate Markdown summary report of scan matches', + ) + + # Input file argument - SCANOSS scan results in JSON format + p_gl_inspect_matches.add_argument( + '-i', + '--input', + required=True, + type=str, + help='Path to SCANOSS scan results file (JSON format) to analyze' + ) + + # Line range prefix for GitLab file navigation + # Enables clickable file references in the generated report that link to specific lines in GitLab + p_gl_inspect_matches.add_argument( + '-lpr', + '--line-range-prefix', + required=True, + type=str, + help='Base URL prefix for GitLab file links with line ranges (e.g., https://gitlab.com/org/project/-/blob/main)' + ) + + # Output file argument - where to save the generated Markdown report + p_gl_inspect_matches.add_argument( + '--output', + '-o', + required=False, + type=str, + help='Output file path for the generated Markdown report (default: stdout)' + ) + # TODO Move to the command call def location # RAW results p_inspect_raw_undeclared.set_defaults(func=inspect_undeclared) @@ -807,6 +869,8 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_inspect_legacy_component_summary.set_defaults(func=inspect_component_summary) # Dependency Track p_inspect_dt_project_violation.set_defaults(func=inspect_dep_track_project_violations) + # GitLab + p_gl_inspect_matches.set_defaults(func=inspect_gitlab_matches) # ========================================================================= # END INSPECT SUBCOMMAND CONFIGURATION @@ -1153,6 +1217,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_inspect_legacy_license_summary, p_inspect_legacy_component_summary, p_inspect_dt_project_violation, + p_gl_inspect_matches, c_provenance, p_folder_scan, p_folder_hash, @@ -1207,7 +1272,11 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) and not args.subparsercmd: parser.parse_args([args.subparser, '--help']) # Force utils helps to be displayed sys.exit(1) - elif (args.subparser in 'inspect') and (args.subparsercmd in ('raw', 'dt')) and (args.subparser_subcmd is None): + elif ( + (args.subparser in 'inspect') + and (args.subparsercmd in ('raw', 'dt', 'glc', 'gitlab')) + and (args.subparser_subcmd is None) + ): parser.parse_args([args.subparser, args.subparsercmd, '--help']) # Force utils helps to be displayed sys.exit(1) args.func(parser, args) # Execute the function associated with the sub-command @@ -1628,6 +1697,11 @@ def convert(parser, args): print_stderr('Producing CSV report...') csvo = CsvOutput(debug=args.debug, output_file=args.output) success = csvo.produce_from_file(args.input) + elif args.format == 'glc-codequality': + if not args.quiet: + print_stderr('Producing GitLab code quality report...') + glc_code_quality = GitLabQualityReport(debug=args.debug, trace=args.trace, quiet=args.quiet) + success = glc_code_quality.produce_from_file(args.input, output_file=args.output) else: print_stderr(f'ERROR: Unknown output format (--format): {args.format}') if not success: @@ -1901,6 +1975,69 @@ def inspect_dep_track_project_violations(parser, args): sys.exit(1) +def inspect_gitlab_matches(parser,args): + """ + Handle GitLab matches the summary inspection command. + + Analyzes SCANOSS scan results and generates a GitLab-compatible Markdown summary + report of component matches. The report includes match details, file locations, + and optionally clickable links to source files in GitLab repositories. + + This command processes SCANOSS scan output and creates human-readable Markdown. + + Parameters + ---------- + parser : ArgumentParser + Command line parser object for help display + args : Namespace + Parsed command line arguments containing: + - input: Path to SCANOSS scan results file (JSON format) to analyze + - line_range_prefix: Base URL prefix for generating GitLab file links with line ranges + (e.g., 'https://gitlab.com/org/project/-/blob/main') + - output: Optional output file path for the generated Markdown report (default: stdout) + - debug: Enable debug output for troubleshooting + - trace: Enable trace-level logging + - quiet: Suppress informational messages + + Notes + ----- + - The output is formatted in Markdown for optimal display in GitLab + - Line range prefix enables clickable file references in the report + - If output is not specified, the report is written to stdout + """ + + if args.input is None: + parser.parse_args([args.subparser, '-h']) + sys.exit(1) + + if args.line_range_prefix is None: + parser.parse_args([args.subparser, '-h']) + sys.exit(1) + + # Initialize output file if specified (create/truncate) + if args.output: + initialise_empty_file(args.output) + + try: + # Create GitLab matches summary generator with configuration + match_summary = MatchSummary( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + scanoss_results_path=args.input, # Path to SCANOSS JSON results + output=args.output, # Output file path or None for stdout + line_range_prefix=args.line_range_prefix, # GitLab URL prefix for file links + ) + + # Execute the summary generation + match_summary.run() + except Exception as e: + # Handle any errors during report generation + print_stderr(e) + if args.debug: + traceback.print_exc() + sys.exit(1) + # ============================================================================= # END INSPECT COMMAND HANDLERS # ============================================================================= diff --git a/src/scanoss/gitlabqualityreport.py b/src/scanoss/gitlabqualityreport.py new file mode 100644 index 00000000..62dc25f4 --- /dev/null +++ b/src/scanoss/gitlabqualityreport.py @@ -0,0 +1,185 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import os +import sys +from dataclasses import dataclass + +from .scanossbase import ScanossBase +from .utils import scanoss_scan_results_utils + + +@dataclass +class Lines: + begin: int + +@dataclass +class Location: + path: str + lines: Lines + +@dataclass +class CodeQuality: + description: str + check_name: str + fingerprint: str + severity: str + location: Location + + def to_dict(self): + """Convert to dictionary for JSON serialization.""" + return { + "description": self.description, + "check_name": self.check_name, + "fingerprint": self.fingerprint, + "severity": self.severity, + "location": { + "path": self.location.path, + "lines": { + "begin": self.location.lines.begin + } + } + } + +class GitLabQualityReport(ScanossBase): + """ + GitLabCodeQuality management class + Handle all interaction with GitLab Code Quality Report formatting + """ + + def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False): + """ + Initialise the GitLabCodeQuality class + """ + super().__init__(debug, trace, quiet) + + + def _get_code_quality(self, file_name: str, result: dict) -> CodeQuality or None: + if not result.get('file_hash'): + self.print_debug(f"Warning: no hash found for result: {result}") + return None + + if result.get('id') == 'file': + description = f"File match found in: {file_name}" + return CodeQuality( + description=description, + check_name=file_name, + fingerprint=result.get('file_hash'), + severity="info", + location=Location( + path=file_name, + lines = Lines( + begin= 1 + ) + ) + ) + + if not result.get('lines'): + self.print_debug(f"Warning: No lines found for result: {result}") + return None + lines = scanoss_scan_results_utils.get_lines(result.get('lines')) + if len(lines) == 0: + self.print_debug(f"Warning: empty lines for result: {result}") + return None + end_line = lines[len(lines) - 1] if len(lines) > 1 else lines[0] + description = f"Snippet found in: {file_name} - lines {lines[0]}-{end_line}" + return CodeQuality( + description=description, + check_name=file_name, + fingerprint=result.get('file_hash'), + severity="info", + location=Location( + path=file_name, + lines=Lines( + begin=lines[0] + ) + ) + ) + + def _write_output(self, data: list[CodeQuality], output_file: str = None) -> bool: + """Write the Gitlab Code Quality Report to output.""" + try: + json_data = [item.to_dict() for item in data] + file = open(output_file, 'w') if output_file else sys.stdout + print(json.dumps(json_data, indent=2), file=file) + if output_file: + file.close() + return True + except Exception as e: + self.print_stderr(f'Error writing output: {str(e)}') + return False + + def _produce_from_json(self, data: dict, output_file: str = None) -> bool: + code_quality = [] + for file_name, results in data.items(): + for result in results: + if not result.get('id'): + self.print_debug(f"Warning: No ID found for result: {result}") + continue + if result.get('id') != 'snippet' and result.get('id') != 'file': + self.print_debug(f"Skipping non-snippet/file match: {result}") + continue + code_quality_item = self._get_code_quality(file_name, result) + if code_quality_item: + code_quality.append(code_quality_item) + else: + self.print_debug(f"Warning: No Code Quality found for result: {result}") + self._write_output(data=code_quality,output_file=output_file) + return True + + def _produce_from_str(self, json_str: str, output_file: str = None) -> bool: + """ + Produce Gitlab Code Quality Report output from input JSON string + :param json_str: input JSON string + :param output_file: Output file (optional) + :return: True if successful, False otherwise + """ + if not json_str: + self.print_stderr('ERROR: No JSON string provided to parse.') + return False + try: + data = json.loads(json_str) + except Exception as e: + self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + return False + return self._produce_from_json(data, output_file) + + + def produce_from_file(self, json_file: str, output_file: str = None) -> bool: + """ + Parse plain/raw input JSON file and produce GitLab Code Quality JSON output + :param json_file: + :param output_file: + :return: True if successful, False otherwise + """ + if not json_file: + self.print_stderr('ERROR: No JSON file provided to parse.') + return False + if not os.path.isfile(json_file): + self.print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') + return False + with open(json_file, 'r') as f: + success = self._produce_from_str(f.read(), output_file) + return success diff --git a/src/scanoss/inspection/dependency_track/project_violation.py b/src/scanoss/inspection/dependency_track/project_violation.py index b1c7597c..c891d76c 100644 --- a/src/scanoss/inspection/dependency_track/project_violation.py +++ b/src/scanoss/inspection/dependency_track/project_violation.py @@ -28,6 +28,7 @@ from ...services.dependency_track_service import DependencyTrackService from ..policy_check import PolicyCheck, PolicyStatus +from ..utils.markdown_utils import generate_jira_table, generate_table # Constants PROCESSING_RETRY_DELAY = 5 # seconds @@ -195,7 +196,7 @@ def _markdown(self, project_violations: list[PolicyViolationDict]) -> Dict[str, Returns: Dictionary with formatted Markdown details and summary """ - return self._md_summary_generator(project_violations, self.generate_table) + return self._md_summary_generator(project_violations, generate_table) def _jira_markdown(self, data: list[PolicyViolationDict]) -> Dict[str, Any]: """ @@ -207,7 +208,7 @@ def _jira_markdown(self, data: list[PolicyViolationDict]) -> Dict[str, Any]: Returns: Dictionary containing Jira markdown formatted results and summary """ - return self._md_summary_generator(data, self.generate_jira_table) + return self._md_summary_generator(data, generate_jira_table) def is_project_updated(self, dt_project: Dict[str, Any]) -> bool: """ diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check.py index decde1aa..cd01972f 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check.py @@ -137,48 +137,6 @@ def _jira_markdown(self, data: list[T]) -> Dict[str, Any]: """ pass - def generate_table(self, headers, rows, centered_columns=None): - """ - Generate a Markdown table. - - :param headers: List of headers for the table. - :param rows: List of rows for the table. - :param centered_columns: List of column indices to be centered. - :return: A string representing the Markdown table. - """ - col_sep = ' | ' - centered_column_set = set(centered_columns or []) - if headers is None: - self.print_stderr('ERROR: Header are no set') - return None - - # Decide which separator to use - def create_separator(index): - if centered_columns is None: - return '-' - return ':-:' if index in centered_column_set else '-' - - # Build the row separator - row_separator = col_sep + col_sep.join(create_separator(index) for index, _ in enumerate(headers)) + col_sep - # build table rows - table_rows = [col_sep + col_sep.join(headers) + col_sep, row_separator] - table_rows.extend(col_sep + col_sep.join(row) + col_sep for row in rows) - return '\n'.join(table_rows) - - def generate_jira_table(self, headers, rows, centered_columns=None): - col_sep = '*|*' - if headers is None: - self.print_stderr('ERROR: Header are no set') - return None - - table_header = '|*' + col_sep.join(headers) + '*|\n' - table = table_header - for row in rows: - if len(headers) == len(row): - table += '|' + '|'.join(row) + '|\n' - - return table - def _get_formatter(self) -> Callable[[List[dict]], Dict[str, Any]] or None: """ Get the appropriate formatter function based on the specified format. diff --git a/src/scanoss/inspection/raw/__init__.py b/src/scanoss/inspection/raw/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/scanoss/inspection/raw/component_summary.py b/src/scanoss/inspection/raw/component_summary.py index 6c337a82..03bddbd8 100644 --- a/src/scanoss/inspection/raw/component_summary.py +++ b/src/scanoss/inspection/raw/component_summary.py @@ -21,13 +21,58 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - import json +from typing import Any +from ..policy_check import T from .raw_base import RawBase class ComponentSummary(RawBase): + + def _json(self, data: dict[str,Any]) -> dict[str,Any]: + """ + Format component summary data as JSON. + + This method returns the component summary data in its original JSON structure + without any transformation. The data can be directly serialized to JSON format. + + :param data: Dictionary containing component summary information including: + - components: List of component-license pairs with status and metadata + - totalComponents: Total number of unique components + - undeclaredComponents: Number of components with 'pending' status + - declaredComponents: Number of components with 'identified' status + - totalFilesDetected: Total count of files where components were detected + - totalFilesUndeclared: Count of files with undeclared components + - totalFilesDeclared: Count of files with declared components + :return: The same data dictionary, ready for JSON serialization + """ + return data + + def _markdown(self, data: list[T]) -> dict[str, Any]: + """ + Format component summary data as Markdown (not yet implemented). + + This method is intended to convert component summary data into a human-readable + Markdown format with tables and formatted sections. + + :param data: List of component summary items to format + :return: Dictionary containing formatted Markdown output + """ + pass + + def _jira_markdown(self, data: list[T]) -> dict[str, Any]: + """ + Format component summary data as Jira-flavored Markdown (not yet implemented). + + This method is intended to convert component summary data into Jira-compatible + Markdown format, which may include Jira-specific syntax for tables and formatting. + + :param data: List of component summary items to format + :return: Dictionary containing Jira-formatted Markdown output + """ + pass + def _get_component_summary_from_components(self, scan_components: list)-> dict: """ Get a component summary from detected components. @@ -84,10 +129,16 @@ def _get_components(self): self._get_components_data(self.results, components) return self._convert_components_to_list(components) + def _format(self, component_summary) -> str: + # TODO: Implement formatter to support dynamic outputs + json_data = self._json(component_summary) + return json.dumps(json_data, indent=2) + def run(self): components = self._get_components() component_summary = self._get_component_summary_from_components(components) - self.print_to_file_or_stdout(json.dumps(component_summary, indent=2), self.output) + output = self._format(component_summary) + self.print_to_file_or_stdout(output, self.output) return component_summary # # End of ComponentSummary Class diff --git a/src/scanoss/inspection/raw/copyleft.py b/src/scanoss/inspection/raw/copyleft.py index d778c2e5..97c25bab 100644 --- a/src/scanoss/inspection/raw/copyleft.py +++ b/src/scanoss/inspection/raw/copyleft.py @@ -27,6 +27,7 @@ from typing import Any, Dict, List from ..policy_check import PolicyStatus +from ..utils.markdown_utils import generate_jira_table, generate_table from .raw_base import RawBase @@ -111,7 +112,7 @@ def _markdown(self, components: list[Component]) -> Dict[str, Any]: :param components: List of components with copyleft licenses :return: Dictionary with formatted Markdown details and summary """ - return self._md_summary_generator(components, self.generate_table) + return self._md_summary_generator(components, generate_table) def _jira_markdown(self, components: list[Component]) -> Dict[str, Any]: """ @@ -120,7 +121,7 @@ def _jira_markdown(self, components: list[Component]) -> Dict[str, Any]: :param components: List of components with copyleft licenses :return: Dictionary with formatted Markdown details and summary """ - return self._md_summary_generator(components, self.generate_jira_table) + return self._md_summary_generator(components, generate_jira_table) def _md_summary_generator(self, components: list[Component], table_generator): """ diff --git a/src/scanoss/inspection/raw/license_summary.py b/src/scanoss/inspection/raw/license_summary.py index bd85c56d..c849ea0a 100644 --- a/src/scanoss/inspection/raw/license_summary.py +++ b/src/scanoss/inspection/raw/license_summary.py @@ -23,7 +23,9 @@ """ import json +from typing import Any +from ..policy_check import T from .raw_base import RawBase @@ -36,6 +38,46 @@ class LicenseSummary(RawBase): information, providing detailed summaries including copyleft analysis and license statistics. """ + def _json(self, data: dict[str,Any]) -> dict[str, Any]: + """ + Format license summary data as JSON. + + This method is intended to return the license summary data in JSON structure + for serialization. The data should include license information with copyleft + analysis and license statistics. + + :param data: List of license summary items to format + :return: Dictionary containing license summary information including: + - licenses: List of detected licenses with SPDX IDs, URLs, and copyleft status + - detectedLicenses: Total number of unique licenses + - detectedLicensesWithCopyleft: Count of licenses marked as copyleft + """ + return data + + def _markdown(self, data: list[T]) -> dict[str, Any]: + """ + Format license summary data as Markdown (not yet implemented). + + This method is intended to convert license summary data into a human-readable + Markdown format with tables and formatted sections. + + :param data: List of license summary items to format + :return: Dictionary containing formatted Markdown output + """ + pass + + def _jira_markdown(self, data: list[T]) -> dict[str, Any]: + """ + Format license summary data as Jira-flavored Markdown (not yet implemented). + + This method is intended to convert license summary data into Jira-compatible + Markdown format, which may include Jira-specific syntax for tables and formatting. + + :param data: List of license summary items to format + :return: Dictionary containing Jira-formatted Markdown output + """ + pass + # Define required license fields as class constants REQUIRED_LICENSE_FIELDS = ['spdxid', 'url', 'copyleft', 'source'] @@ -131,10 +173,16 @@ def _get_components(self): self._get_dependencies_data(self.results, components) return self._convert_components_to_list(components) + def _format(self, license_summary) -> str: + # TODO: Implement formatter to support dynamic outputs + json_data = self._json(license_summary) + return json.dumps(json_data, indent=2) + def run(self): components = self._get_components() license_summary = self._get_licenses_summary_from_components(components) - self.print_to_file_or_stdout(json.dumps(license_summary, indent=2), self.output) + output = self._format(license_summary) + self.print_to_file_or_stdout(output, self.output) return license_summary # # End of LicenseSummary Class diff --git a/src/scanoss/inspection/raw/match_summary.py b/src/scanoss/inspection/raw/match_summary.py new file mode 100644 index 00000000..645824bb --- /dev/null +++ b/src/scanoss/inspection/raw/match_summary.py @@ -0,0 +1,290 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +from dataclasses import dataclass + +from ...scanossbase import ScanossBase +from ...utils import scanoss_scan_results_utils +from ..utils.file_utils import load_json_file +from ..utils.markdown_utils import generate_table + + +@dataclass +class MatchSummaryItem: + """ + Represents a single match entry in the SCANOSS results. + + This data class encapsulates all the relevant information about a component + match found during scanning, including file location, license details, and + match quality metrics. + """ + file: str + file_url: str + license: str + similarity: str + purl: str + purl_url: str + version: str + lines: str + + +@dataclass +class ComponentMatchSummary: + """ + Container for categorized SCANOSS match results. + + Organizes matches into two categories: full file matches and snippet matches. + This separation allows for different presentation and analysis of match types. + """ + files: list[MatchSummaryItem] + snippet: list[MatchSummaryItem] + +class MatchSummary(ScanossBase): + """ + Generates Markdown summaries from SCANOSS scan results. + + This class processes SCANOSS scan results and creates human-readable Markdown + reports with collapsible sections for file and snippet matches. The reports + include clickable links to files when a line range + prefix is provided. + """ + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + line_range_prefix: str = None, + scanoss_results_path: str = None, + output: str = None, + ): + """ + Initialize the Matches Summary generator. + + :param debug: Enable debug output for troubleshooting + :param trace: Enable trace-level logging for detailed execution tracking + :param quiet: Suppress informational messages + :param line_range_prefix: Base URL prefix for GitLab file links with line ranges + (e.g., 'https://gitlab.com/org/project/-/blob/main') + :param scanoss_results_path: Path to SCANOSS scan results file in JSON format + :param output: Output file path for the generated Markdown report (default: stdout) + """ + super().__init__(debug=debug, trace=trace, quiet=quiet) + self.scanoss_results_path = scanoss_results_path + self.line_range_prefix = line_range_prefix + self.output = output + + + def _get_match_summary_item(self, file_name: str, result: dict) -> MatchSummaryItem: + """ + Create a MatchSummaryItem from a single scan result. + + Processes a SCANOSS scan result and creates a MatchSummaryItem with appropriate + file URLs, license information, and line ranges. Handles both snippet matches + (with specific line ranges) and file matches (entire file). + + :param file_name: Name of the scanned file (relative path in the repository) + :param result: SCANOSS scan result dictionary containing match details + :return: Populated match summary item with all relevant information + """ + if result.get('id') == "snippet": + # Snippet match: create URL with line range anchor + lines = scanoss_scan_results_utils.get_lines(result.get('lines')) + end_line = lines[len(lines) - 1] if len(lines) > 1 else lines[0] + file_url = f"{self.line_range_prefix}/{file_name}#L{lines[0]}-L{end_line}" + return MatchSummaryItem( + file_url=file_url, + file=file_name, + license=result.get('licenses')[0].get('name'), + similarity=result.get('matched'), + purl=result.get('purl')[0], + purl_url=result.get('url'), + version=result.get('version'), + lines=f"{lines[0]}-{lines[len(lines) - 1] if len(lines) > 1 else lines[0]}" + ) + # File match: create URL without line range + return MatchSummaryItem( + file=file_name, + file_url=f"{self.line_range_prefix}/{file_name}", + license=result.get('licenses')[0].get('name'), + similarity=result.get('matched'), + purl=result.get('purl')[0], + purl_url=result.get('url'), + version=result.get('version'), + lines="all" + ) + + def _validate_result(self, file_name: str, result: dict) -> bool: + """ + Validate that a scan result has all required fields. + + :param file_name: Name of the file being validated + :param result: The scan result to validate + :return: True if valid, False otherwise + """ + validations = [ + ('id', 'No id found'), + ('lines', 'No lines found'), + ('purl', 'No purl found'), + ('licenses', 'No licenses found'), + ('version', 'No version found'), + ('matched', 'No matched found'), + ('url', 'No url found'), + ] + + for field, error_msg in validations: + if not result.get(field): + self.print_debug(f'ERROR: {error_msg} for file {file_name}') + return False + + # Additional validation for non-empty lists + if len(result.get('purl')) == 0: + self.print_debug(f'ERROR: No purl found for file {file_name}') + return False + if len(result.get('licenses')) == 0: + self.print_debug(f'ERROR: Empty licenses list for file {file_name}') + return False + + return True + + def _get_matches_summary(self) -> ComponentMatchSummary: + """ + Parse SCANOSS scan results and create categorized match summaries. + + Loads the SCANOSS scan results file and processes each match, validating + required fields and categorizing matches into file matches and snippet matches. + Skips invalid or incomplete results with debug messages. + """ + # Load scan results from JSON file + scan_results = load_json_file(self.scanoss_results_path) + gitlab_matches_summary = ComponentMatchSummary(files=[], snippet=[]) + + # Process each file and its results + for file_name, results in scan_results.items(): + for result in results: + # Skip non-matches + if result.get('id') == "none": + self.print_debug(f'Skipping non-match for file {file_name}') + continue + + # Validate required fields + if not self._validate_result(file_name, result): + continue + + # Create summary item and categorize by match type + summary_item = self._get_match_summary_item(file_name, result) + if result.get('id') == "snippet": + gitlab_matches_summary.snippet.append(summary_item) + else: + gitlab_matches_summary.files.append(summary_item) + + return gitlab_matches_summary + + + def _markdown(self, gitlab_matches_summary: ComponentMatchSummary) -> str: + """ + Generate Markdown from match summaries. + + Creates a formatted Markdown document with collapsible sections for file + and snippet matches. + + :param gitlab_matches_summary: Container with categorized file and snippet matches to format + :return: Complete Markdown document with formatted match tables + """ + + if len(gitlab_matches_summary.files) == 0 and len(gitlab_matches_summary.snippet) == 0: + return "" + + # Define table headers + file_match_headers = ['File', 'License', 'Similarity', 'PURL', 'Version'] + snippet_match_headers = ['File', 'License', 'Similarity', 'PURL', 'Version', 'Lines'] + # Build file matches table + file_match_rows = [] + for file_match in gitlab_matches_summary.files: + row = [ + f"[{file_match.file}]({file_match.file_url})", + file_match.license, + file_match.similarity, + f"[{file_match.purl}]({file_match.purl_url})", + file_match.version, + ] + file_match_rows.append(row) + file_match_table = generate_table(file_match_headers, file_match_rows) + + # Build snippet matches table + snippet_match_rows = [] + for snippet_match in gitlab_matches_summary.snippet: + row = [ + f"[{snippet_match.file}]({snippet_match.file_url})", + snippet_match.license, + snippet_match.similarity, + f"[{snippet_match.purl}]({snippet_match.purl_url})", + snippet_match.version, + snippet_match.lines + ] + snippet_match_rows.append(row) + snippet_match_table = generate_table(snippet_match_headers, snippet_match_rows) + + # Assemble complete Markdown document + markdown = "" + markdown += "### SCANOSS Match Summary\n\n" + + # File matches section (collapsible) + markdown += "
\n" + markdown += "File Match Summary\n\n" + markdown += file_match_table + markdown += "\n
\n" + + # Snippet matches section (collapsible) + markdown += "
\n" + markdown += "Snippet Match Summary\n\n" + markdown += snippet_match_table + markdown += "\n
\n" + + return markdown + + def run(self): + """ + Execute the matches summary generation process. + + This is the main entry point for generating the matches summary report. + It orchestrates the entire workflow: + 1. Loads and parses SCANOSS scan results + 2. Validates and categorizes matches + 3. Generates Markdown report + 4. Outputs to file or stdout + """ + # Load and process scan results into categorized matches + matches = self._get_matches_summary() + + # Format matches as GitLab-compatible Markdown + matches_md = self._markdown(matches) + if matches_md == "": + self.print_stdout("No matches found.") + return + # Output to file or stdout + self.print_to_file_or_stdout(matches_md, self.output) + + + diff --git a/src/scanoss/inspection/raw/raw_base.py b/src/scanoss/inspection/raw/raw_base.py index 0bae631e..8a1a6f8d 100644 --- a/src/scanoss/inspection/raw/raw_base.py +++ b/src/scanoss/inspection/raw/raw_base.py @@ -22,13 +22,12 @@ THE SOFTWARE. """ -import json -import os.path from abc import abstractmethod from enum import Enum from typing import Any, Dict, TypeVar from ..policy_check import PolicyCheck +from ..utils.file_utils import load_json_file from ..utils.license_utils import LicenseUtil @@ -313,15 +312,11 @@ def _load_input_file(self): Returns: Dict[str, Any]: The parsed JSON data """ - if not os.path.exists(self.filepath): - self.print_stderr(f'ERROR: The file "{self.filepath}" does not exist.') - return None - with open(self.filepath, 'r') as jsonfile: - try: - return json.load(jsonfile) - except Exception as e: + try: + return load_json_file(self.filepath) + except Exception as e: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') - return None + return None def _convert_components_to_list(self, components: dict): if components is None: diff --git a/src/scanoss/inspection/raw/undeclared_component.py b/src/scanoss/inspection/raw/undeclared_component.py index 948dd907..a7e32dac 100644 --- a/src/scanoss/inspection/raw/undeclared_component.py +++ b/src/scanoss/inspection/raw/undeclared_component.py @@ -27,6 +27,7 @@ from typing import Any, Dict, List from ..policy_check import PolicyStatus +from ..utils.markdown_utils import generate_jira_table, generate_table from .raw_base import RawBase @@ -193,7 +194,7 @@ def _markdown(self, components: list[Component]) -> Dict[str, Any]: for component in component_licenses: rows.append([component.get('purl'), component.get('spdxid')]) return { - 'details': f'### Undeclared components\n{self.generate_table(headers, rows)}\n', + 'details': f'### Undeclared components\n{generate_table(headers, rows)}\n', 'summary': self._get_summary(component_licenses), } @@ -211,7 +212,7 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: for component in component_licenses: rows.append([component.get('purl'), component.get('spdxid')]) return { - 'details': f'{self.generate_jira_table(headers, rows)}', + 'details': f'{generate_jira_table(headers, rows)}', 'summary': self._get_jira_summary(component_licenses), } diff --git a/src/scanoss/inspection/utils/file_utils.py b/src/scanoss/inspection/utils/file_utils.py new file mode 100644 index 00000000..a7e5de41 --- /dev/null +++ b/src/scanoss/inspection/utils/file_utils.py @@ -0,0 +1,44 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import os + + +def load_json_file(file_path: str) -> dict: + """ + Load the file + + :param file_path: file path to the JSON file + + Returns: + Dict[str, Any]: The parsed JSON data + """ + if not os.path.exists(file_path): + raise ValueError(f'The file "{file_path}" does not exist.') + with open(file_path, 'r') as jsonfile: + try: + return json.load(jsonfile) + except Exception as e: + raise ValueError(f'ERROR: Problem parsing input JSON: {e}') \ No newline at end of file diff --git a/src/scanoss/inspection/utils/markdown_utils.py b/src/scanoss/inspection/utils/markdown_utils.py new file mode 100644 index 00000000..0ce47a26 --- /dev/null +++ b/src/scanoss/inspection/utils/markdown_utils.py @@ -0,0 +1,63 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +def generate_table(headers, rows, centered_columns=None): + """ + Generate a Markdown table. + + :param headers: List of headers for the table. + :param rows: List of rows for the table. + :param centered_columns: List of column indices to be centered. + :return: A string representing the Markdown table. + """ + col_sep = ' | ' + centered_column_set = set(centered_columns or []) + if headers is None: + return None + + # Decide which separator to use + def create_separator(index): + if centered_columns is None: + return '-' + return ':-:' if index in centered_column_set else '-' + + # Build the row separator + row_separator = col_sep + col_sep.join(create_separator(index) for index, _ in enumerate(headers)) + col_sep + # build table rows + table_rows = [col_sep + col_sep.join(headers) + col_sep, row_separator] + table_rows.extend(col_sep + col_sep.join(row) + col_sep for row in rows) + return '\n'.join(table_rows) + +def generate_jira_table(headers, rows, centered_columns=None): + col_sep = '*|*' + if headers is None: + return None + + table_header = '|*' + col_sep.join(headers) + '*|\n' + table = table_header + for row in rows: + if len(headers) == len(row): + table += '|' + '|'.join(row) + '|\n' + + return table \ No newline at end of file diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index 2e516780..eb4bd726 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -158,6 +158,7 @@ def _build_root_node( filtered_files.sort() bar = Bar('Hashing files...', max=len(filtered_files)) + full_file_path = '' for file_path in filtered_files: try: file_path_obj = Path(file_path) if isinstance(file_path, str) else file_path diff --git a/src/scanoss/utils/scanoss_scan_results_utils.py b/src/scanoss/utils/scanoss_scan_results_utils.py new file mode 100644 index 00000000..a9ac1fbb --- /dev/null +++ b/src/scanoss/utils/scanoss_scan_results_utils.py @@ -0,0 +1,41 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +def get_lines(lines: str) -> list: + """ + Parse line range string into a list of line numbers. + + Converts SCANOSS line notation (e.g., '10-20,25-30') into a flat list + of individual line numbers for processing. + + :param lines: Comma-separated line ranges in SCANOSS format (e.g., '10-20,25-30') + :return: Flat list of all line numbers extracted from the ranges + """ + lines_list = [] + lines = lines.split(',') + for line in lines: + line_parts = line.split('-') + for part in line_parts: + lines_list.append(int(part)) + return lines_list \ No newline at end of file From 5d04af963b9bc05ed4e61b5a0f464a0306c9fabb Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Mon, 27 Oct 2025 11:46:21 -0300 Subject: [PATCH 398/489] chore:upgrades version to v1.39.0 --- CHANGELOG.md | 2 +- src/scanoss/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aef5fbb2..9c697604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... -## [1.39.0] - 2025-10-24 +## [1.39.0] - 2025-10-27 ### Added - Added `glc-codequality` format to convert subcommand - Added `inspect gitlab matches` subcommand to generate GitLab-compatible Markdown match summary from SCANOSS scan results diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 23ddbcbd..2f6ae6ce 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.38.0' +__version__ = '1.39.0' From 8c44231cece0cf2af2b48c988e2e665e834c766f Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Wed, 29 Oct 2025 15:17:19 +0100 Subject: [PATCH 399/489] [SP-3549] feat: add rest support for folder scan command (#160) --- src/scanoss/cli.py | 16 +++++++--------- src/scanoss/scanners/scanner_hfh.py | 4 +++- src/scanoss/scanossgrpc.py | 7 +++++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 0542ebeb..c5dd20a3 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -796,7 +796,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Timeout (in seconds) for API communication (optional - default 300 sec)', ) - # ============================================================================== # GitLab Integration Parser # ============================================================================== @@ -830,11 +829,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 # Input file argument - SCANOSS scan results in JSON format p_gl_inspect_matches.add_argument( - '-i', - '--input', - required=True, - type=str, - help='Path to SCANOSS scan results file (JSON format) to analyze' + '-i', '--input', required=True, type=str, help='Path to SCANOSS scan results file (JSON format) to analyze' ) # Line range prefix for GitLab file navigation @@ -844,7 +839,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--line-range-prefix', required=True, type=str, - help='Base URL prefix for GitLab file links with line ranges (e.g., https://gitlab.com/org/project/-/blob/main)' + help='Base URL prefix for GitLab file links with line ranges (e.g., https://gitlab.com/org/project/-/blob/main)', ) # Output file argument - where to save the generated Markdown report @@ -853,7 +848,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '-o', required=False, type=str, - help='Output file path for the generated Markdown report (default: stdout)' + help='Output file path for the generated Markdown report (default: stdout)', ) # TODO Move to the command call def location @@ -1189,6 +1184,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_search, c_versions, c_licenses, + p_folder_scan, ]: p.add_argument('--grpc', action='store_true', default=True, help='Use gRPC (default)') p.add_argument('--rest', action='store_true', dest='rest', help='Use REST instead of gRPC') @@ -1975,7 +1971,7 @@ def inspect_dep_track_project_violations(parser, args): sys.exit(1) -def inspect_gitlab_matches(parser,args): +def inspect_gitlab_matches(parser, args): """ Handle GitLab matches the summary inspection command. @@ -2038,6 +2034,7 @@ def inspect_gitlab_matches(parser,args): traceback.print_exc() sys.exit(1) + # ============================================================================= # END INSPECT COMMAND HANDLERS # ============================================================================= @@ -2673,6 +2670,7 @@ def folder_hashing_scan(parser, args): depth=args.depth, recursive_threshold=args.recursive_threshold, min_accepted_score=args.min_accepted_score, + use_grpc=args.grpc, ) if scanner.scan(): diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 2418d4db..7ac64630 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -63,6 +63,7 @@ def __init__( # noqa: PLR0913 depth: int = DEFAULT_HFH_DEPTH, recursive_threshold: float = DEFAULT_HFH_RECURSIVE_THRESHOLD, min_accepted_score: float = DEFAULT_HFH_MIN_ACCEPTED_SCORE, + use_grpc: bool = False, ): """ Initialize the ScannerHFH. @@ -107,6 +108,7 @@ def __init__( # noqa: PLR0913 self.rank_threshold = rank_threshold self.recursive_threshold = recursive_threshold self.min_accepted_score = min_accepted_score + self.use_grpc = use_grpc def scan(self) -> Optional[Dict]: """ @@ -134,7 +136,7 @@ def spin(): spinner_thread.start() try: - response = self.client.folder_hash_scan(hfh_request) + response = self.client.folder_hash_scan(hfh_request, self.use_grpc) if response: self.scan_results = response finally: diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 5cb73978..490ae72a 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -426,21 +426,24 @@ def get_component_versions_json(self, search: dict, use_grpc: Optional[bool] = N use_grpc=use_grpc, ) - def folder_hash_scan(self, request: Dict) -> Optional[Dict]: + def folder_hash_scan(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: """ Client function to call the rpc for Folder Hashing Scan Args: request (Dict): Folder Hash Request + use_grpc (Optional[bool]): Whether to use gRPC or REST API Returns: Optional[Dict]: Folder Hash Response, or None if the request was not succesfull """ - return self._call_rpc( + return self._call_api( + 'scanning.FolderHashScan', self.scanning_stub.FolderHashScan, request, HFHRequest, 'Sending folder hash scan data (rqId: {rqId})...', + use_grpc=use_grpc, ) def _call_api( From 4d5482e3c50a4f5f4b24a2d69daa08e5a9068070 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 29 Oct 2025 15:20:58 +0100 Subject: [PATCH 400/489] chore: bump version, update changelog --- CHANGELOG.md | 7 ++++++- src/scanoss/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c697604..67a32b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.40.0] - 2025-10-29 +### Added +- Add support for `--rest` to `folder-scan` command + ## [1.39.0] - 2025-10-27 ### Added - Added `glc-codequality` format to convert subcommand @@ -710,4 +714,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.37.0]: https://github.com/scanoss/scanoss.py/compare/v1.36.0...v1.37.0 [1.37.1]: https://github.com/scanoss/scanoss.py/compare/v1.37.0...v1.37.1 [1.38.0]: https://github.com/scanoss/scanoss.py/compare/v1.37.1...v1.38.0 -[1.39.0]: https://github.com/scanoss/scanoss.py/compare/v1.38.0...v1.39.0 \ No newline at end of file +[1.39.0]: https://github.com/scanoss/scanoss.py/compare/v1.38.0...v1.39.0 +[1.40.0]: https://github.com/scanoss/scanoss.py/compare/v1.39.0...v1.40.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 2f6ae6ce..13f9d8ab 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.39.0' +__version__ = '1.40.0' From 4060c9f3c254711045d1fccc4b3ea29e08201a81 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Wed, 29 Oct 2025 11:56:03 -0300 Subject: [PATCH 401/489] Chore/refactor inspect module * chore:SP-3607 refactor inspect module * chore: Adds local linter to Makefile --- CHANGELOG.md | 14 ++ Makefile | 15 +++ src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 17 ++- src/scanoss/gitlabqualityreport.py | 37 +++++- .../{raw => policy_check}/__init__.py | 0 .../policy_check/dependency_track/__init__.py | 0 .../dependency_track/project_violation.py | 48 +++---- .../{ => policy_check}/policy_check.py | 40 +++--- .../policy_check/scanoss/__init__.py | 0 .../{raw => policy_check/scanoss}/copyleft.py | 70 +++++----- .../scanoss}/undeclared_component.py | 59 +++++---- src/scanoss/inspection/summary/__init__.py | 0 .../{raw => summary}/component_summary.py | 43 ++++-- .../{raw => summary}/license_summary.py | 90 ++++++------- .../{raw => summary}/match_summary.py | 51 +++++++ .../scan_result_processor.py} | 73 ++++------- tests/test_policy_inspect.py | 124 +++++++++--------- tools/linter.sh | 50 +++++++ 19 files changed, 449 insertions(+), 284 deletions(-) rename src/scanoss/inspection/{raw => policy_check}/__init__.py (100%) create mode 100644 src/scanoss/inspection/policy_check/dependency_track/__init__.py rename src/scanoss/inspection/{ => policy_check}/dependency_track/project_violation.py (93%) rename src/scanoss/inspection/{ => policy_check}/policy_check.py (86%) create mode 100644 src/scanoss/inspection/policy_check/scanoss/__init__.py rename src/scanoss/inspection/{raw => policy_check/scanoss}/copyleft.py (80%) rename src/scanoss/inspection/{raw => policy_check/scanoss}/undeclared_component.py (85%) create mode 100644 src/scanoss/inspection/summary/__init__.py rename src/scanoss/inspection/{raw => summary}/component_summary.py (81%) rename src/scanoss/inspection/{raw => summary}/license_summary.py (89%) rename src/scanoss/inspection/{raw => summary}/match_summary.py (82%) rename src/scanoss/inspection/{raw/raw_base.py => utils/scan_result_processor.py} (88%) create mode 100755 tools/linter.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 67a32b73..a19331d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [1.40.1] - 2025-10-29 +### Changed +- Refactored inspect module structure for better organization + - Reorganized inspection modules into `policy_check` and `summary` subdirectories + - Moved copyleft and undeclared component checks to `policy_check/scanoss/` + - Moved component, license, and match summaries to `summary/` + - Moved Dependency Track policy checks to `policy_check/dependency_track/` + - Extracted common scan result processing logic into `ScanResultProcessor` utility class + - Improved type safety with `PolicyOutput` named tuple for policy check results + - Made `PolicyCheck` class explicitly abstract with ABC +### Added +- Added Makefile targets for running ruff linter (`linter`, `linter-fix`, `linter-docker`, `linter-docker-fix`) + ## [1.40.0] - 2025-10-29 ### Added - Add support for `--rest` to `folder-scan` command @@ -716,3 +729,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.38.0]: https://github.com/scanoss/scanoss.py/compare/v1.37.1...v1.38.0 [1.39.0]: https://github.com/scanoss/scanoss.py/compare/v1.38.0...v1.39.0 [1.40.0]: https://github.com/scanoss/scanoss.py/compare/v1.39.0...v1.40.0 +[1.40.1]: https://github.com/scanoss/scanoss.py/compare/v1.40.0...v1.40.1 \ No newline at end of file diff --git a/Makefile b/Makefile index ebde3c57..9c13989a 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,9 @@ dev_setup: date_time_clean ## Setup Python dev env for the current user @echo "Setting up dev env for the current user..." pip3 install -e . +dev_install: ## Install dev dependencies + pip3 install -r requirements-dev.txt + dev_uninstall: ## Uninstall Python dev setup for the current user @echo "Uninstalling dev env..." pip3 uninstall -y scanoss @@ -50,6 +53,18 @@ publish_test: ## Publish the Python package to TestPyPI @echo "Publishing package to TestPyPI..." twine upload --repository testpypi dist/* +lint-docker: ## Run ruff linter with docker + @./tools/linter.sh --docker + +lint-docker-fix: ## Run ruff linter with docker and auto-fix + @./tools/linter.sh --docker --fix + +lint: ## Run ruff linter locally + @./tools/linter.sh + +lint-fix: ## Run ruff linter locally with auto-fix + @./tools/linter.sh --fix + publish: ## Publish Python package to PyPI @echo "Publishing package to PyPI..." twine upload dist/* diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 13f9d8ab..694457df 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.40.0' +__version__ = '1.40.1' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index c5dd20a3..824c5133 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -35,12 +35,6 @@ from scanoss.cryptography import Cryptography, create_cryptography_config_from_args from scanoss.delta import Delta from scanoss.export.dependency_track import DependencyTrackExporter -from scanoss.inspection.dependency_track.project_violation import ( - DependencyTrackProjectViolationPolicyCheck, -) -from scanoss.inspection.raw.component_summary import ComponentSummary -from scanoss.inspection.raw.license_summary import LicenseSummary -from scanoss.inspection.raw.match_summary import MatchSummary from scanoss.scanners.container_scanner import ( DEFAULT_SYFT_COMMAND, DEFAULT_SYFT_TIMEOUT, @@ -75,8 +69,14 @@ from .cyclonedx import CycloneDx from .filecount import FileCount from .gitlabqualityreport import GitLabQualityReport -from .inspection.raw.copyleft import Copyleft -from .inspection.raw.undeclared_component import UndeclaredComponent +from .inspection.policy_check.dependency_track.project_violation import ( + DependencyTrackProjectViolationPolicyCheck, +) +from .inspection.policy_check.scanoss.copyleft import Copyleft +from .inspection.policy_check.scanoss.undeclared_component import UndeclaredComponent +from .inspection.summary.component_summary import ComponentSummary +from .inspection.summary.license_summary import LicenseSummary +from .inspection.summary.match_summary import MatchSummary from .results import Results from .scancodedeps import ScancodeDeps from .scanner import FAST_WINNOWING, Scanner @@ -1753,7 +1753,6 @@ def inspect_copyleft(parser, args): exclude=args.exclude, # Licenses to ignore explicit=args.explicit, # Explicit license list ) - # Execute inspection and exit with appropriate status code status, _ = i_copyleft.run() sys.exit(status) diff --git a/src/scanoss/gitlabqualityreport.py b/src/scanoss/gitlabqualityreport.py index 62dc25f4..1a1b8ec6 100644 --- a/src/scanoss/gitlabqualityreport.py +++ b/src/scanoss/gitlabqualityreport.py @@ -74,16 +74,21 @@ def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False Initialise the GitLabCodeQuality class """ super().__init__(debug, trace, quiet) + self.print_trace(f"GitLabQualityReport initialized with debug={debug}, trace={trace}, quiet={quiet}") def _get_code_quality(self, file_name: str, result: dict) -> CodeQuality or None: + self.print_trace(f"_get_code_quality called for file: {file_name}") + self.print_trace(f"Processing result: {result}") + if not result.get('file_hash'): self.print_debug(f"Warning: no hash found for result: {result}") return None if result.get('id') == 'file': + self.print_debug(f"Processing file match for: {file_name}") description = f"File match found in: {file_name}" - return CodeQuality( + code_quality = CodeQuality( description=description, check_name=file_name, fingerprint=result.get('file_hash'), @@ -95,17 +100,21 @@ def _get_code_quality(self, file_name: str, result: dict) -> CodeQuality or None ) ) ) + self.print_trace(f"Created file CodeQuality object: {code_quality}") + return code_quality if not result.get('lines'): self.print_debug(f"Warning: No lines found for result: {result}") return None lines = scanoss_scan_results_utils.get_lines(result.get('lines')) + self.print_trace(f"Extracted lines: {lines}") if len(lines) == 0: self.print_debug(f"Warning: empty lines for result: {result}") return None end_line = lines[len(lines) - 1] if len(lines) > 1 else lines[0] description = f"Snippet found in: {file_name} - lines {lines[0]}-{end_line}" - return CodeQuality( + self.print_debug(f"Processing snippet match for: {file_name}, lines: {lines[0]}-{end_line}") + code_quality = CodeQuality( description=description, check_name=file_name, fingerprint=result.get('file_hash'), @@ -117,35 +126,47 @@ def _get_code_quality(self, file_name: str, result: dict) -> CodeQuality or None ) ) ) + self.print_trace(f"Created snippet CodeQuality object: {code_quality}") + return code_quality def _write_output(self, data: list[CodeQuality], output_file: str = None) -> bool: """Write the Gitlab Code Quality Report to output.""" + self.print_trace(f"_write_output called with {len(data)} items, output_file: {output_file}") try: json_data = [item.to_dict() for item in data] + self.print_trace(f"JSON data: {json_data}") file = open(output_file, 'w') if output_file else sys.stdout print(json.dumps(json_data, indent=2), file=file) if output_file: file.close() + self.print_debug(f"Wrote output to file: {output_file}") + else: + self.print_debug("Wrote output to 'stdout'") return True except Exception as e: self.print_stderr(f'Error writing output: {str(e)}') return False def _produce_from_json(self, data: dict, output_file: str = None) -> bool: + self.print_trace(f"_produce_from_json called with output_file: {output_file}") + self.print_debug(f"Processing {len(data)} files from JSON data") code_quality = [] for file_name, results in data.items(): + self.print_trace(f"Processing file: {file_name} with {len(results)} results") for result in results: if not result.get('id'): self.print_debug(f"Warning: No ID found for result: {result}") continue if result.get('id') != 'snippet' and result.get('id') != 'file': - self.print_debug(f"Skipping non-snippet/file match: {result}") + self.print_debug(f"Skipping non-snippet/file match: {file_name}, id: '{result['id']}'") continue code_quality_item = self._get_code_quality(file_name, result) if code_quality_item: code_quality.append(code_quality_item) + self.print_trace(f"Added code quality item for {file_name}") else: self.print_debug(f"Warning: No Code Quality found for result: {result}") + self.print_debug(f"Generated {len(code_quality)} code quality items") self._write_output(data=code_quality,output_file=output_file) return True @@ -156,11 +177,15 @@ def _produce_from_str(self, json_str: str, output_file: str = None) -> bool: :param output_file: Output file (optional) :return: True if successful, False otherwise """ + self.print_trace(f"_produce_from_str called with output_file: {output_file}") if not json_str: self.print_stderr('ERROR: No JSON string provided to parse.') return False + self.print_debug(f"Parsing JSON string of length: {len(json_str)}") try: data = json.loads(json_str) + self.print_debug("Successfully parsed JSON data") + self.print_trace(f"Parsed data structure: {type(data)}") except Exception as e: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return False @@ -174,12 +199,16 @@ def produce_from_file(self, json_file: str, output_file: str = None) -> bool: :param output_file: :return: True if successful, False otherwise """ + self.print_trace(f"produce_from_file called with json_file: {json_file}, output_file: {output_file}") + self.print_debug(f"Input JSON file: {json_file}, output_file: {output_file}") if not json_file: self.print_stderr('ERROR: No JSON file provided to parse.') return False if not os.path.isfile(json_file): self.print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}') return False + self.print_debug(f"Reading JSON file: {json_file}") with open(json_file, 'r') as f: - success = self._produce_from_str(f.read(), output_file) + json_content = f.read() + success = self._produce_from_str(json_content, output_file) return success diff --git a/src/scanoss/inspection/raw/__init__.py b/src/scanoss/inspection/policy_check/__init__.py similarity index 100% rename from src/scanoss/inspection/raw/__init__.py rename to src/scanoss/inspection/policy_check/__init__.py diff --git a/src/scanoss/inspection/policy_check/dependency_track/__init__.py b/src/scanoss/inspection/policy_check/dependency_track/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/scanoss/inspection/dependency_track/project_violation.py b/src/scanoss/inspection/policy_check/dependency_track/project_violation.py similarity index 93% rename from src/scanoss/inspection/dependency_track/project_violation.py rename to src/scanoss/inspection/policy_check/dependency_track/project_violation.py index c891d76c..e9d78667 100644 --- a/src/scanoss/inspection/dependency_track/project_violation.py +++ b/src/scanoss/inspection/policy_check/dependency_track/project_violation.py @@ -26,9 +26,9 @@ from datetime import datetime from typing import Any, Dict, List, Optional, TypedDict -from ...services.dependency_track_service import DependencyTrackService -from ..policy_check import PolicyCheck, PolicyStatus -from ..utils.markdown_utils import generate_jira_table, generate_table +from ....services.dependency_track_service import DependencyTrackService +from ...utils.markdown_utils import generate_jira_table, generate_table +from ..policy_check import PolicyCheck, PolicyOutput, PolicyStatus # Constants PROCESSING_RETRY_DELAY = 5 # seconds @@ -171,7 +171,7 @@ def __init__( # noqa: PLR0913 self.url = url.strip().rstrip('/') if url else None self.dep_track_service = DependencyTrackService(self.api_key, self.url, debug=debug, trace=trace, quiet=quiet) - def _json(self, project_violations: list[PolicyViolationDict]) -> Dict[str, Any]: + def _json(self, project_violations: list[PolicyViolationDict]) -> PolicyOutput: """ Format project violations as JSON. @@ -181,12 +181,12 @@ def _json(self, project_violations: list[PolicyViolationDict]) -> Dict[str, Any] Returns: Dictionary containing JSON formatted results and summary """ - return { - "details": json.dumps(project_violations, indent=2), - "summary": f'{len(project_violations)} policy violations were found.\n', - } + return PolicyOutput( + details= json.dumps(project_violations, indent=2), + summary= f'{len(project_violations)} policy violations were found.\n', + ) - def _markdown(self, project_violations: list[PolicyViolationDict]) -> Dict[str, Any]: + def _markdown(self, project_violations: list[PolicyViolationDict]) -> PolicyOutput: """ Format Dependency Track violations to Markdown format. @@ -198,7 +198,7 @@ def _markdown(self, project_violations: list[PolicyViolationDict]) -> Dict[str, """ return self._md_summary_generator(project_violations, generate_table) - def _jira_markdown(self, data: list[PolicyViolationDict]) -> Dict[str, Any]: + def _jira_markdown(self, data: list[PolicyViolationDict]) -> PolicyOutput: """ Format project violations for Jira Markdown. @@ -357,8 +357,7 @@ def _set_project_id(self) -> None: self.print_stderr(f'Error: Failed to get project uuid from: {dt_project}') raise ValueError(f'Error: Project {self.project_name}@{self.project_version} does not have a valid UUID') - @staticmethod - def _sort_project_violations(violations: List[PolicyViolationDict]) -> List[PolicyViolationDict]: + def _sort_project_violations(self,violations: List[PolicyViolationDict]) -> List[PolicyViolationDict]: """ Sort project violations by priority. @@ -377,7 +376,7 @@ def _sort_project_violations(violations: List[PolicyViolationDict]) -> List[Poli key=lambda x: -type_priority.get(x.get('type', 'OTHER'), 1) ) - def _md_summary_generator(self, project_violations: list[PolicyViolationDict], table_generator): + def _md_summary_generator(self, project_violations: list[PolicyViolationDict], table_generator) -> PolicyOutput: """ Generates a Markdown summary of project policy violations. @@ -396,10 +395,10 @@ def _md_summary_generator(self, project_violations: list[PolicyViolationDict], t """ if project_violations is None: self.print_stderr('Warning: No project violations found. Returning empty results.') - return { - "details": "h3. Dependency Track Project Violations\n\nNo policy violations found.\n", - "summary": "0 policy violations were found.\n", - } + return PolicyOutput( + details= "h3. Dependency Track Project Violations\n\nNo policy violations found.\n", + summary= "0 policy violations were found.\n", + ) headers = ['State', 'Risk Type', 'Policy Name', 'Component', 'Date'] c_cols = [0, 1] rows: List[List[str]] = [] @@ -424,11 +423,11 @@ def _md_summary_generator(self, project_violations: list[PolicyViolationDict], t ] rows.append(row) # End for loop - return { - "details": f'### Dependency Track Project Violations\n{table_generator(headers, rows, c_cols)}\n\n' + return PolicyOutput( + details= f'### Dependency Track Project Violations\n{table_generator(headers, rows, c_cols)}\n\n' f'View project in Dependency Track [here]({self.url}/projects/{self.project_id}).\n', - "summary": f'{len(project_violations)} policy violations were found.\n' - } + summary= f'{len(project_violations)} policy violations were found.\n' + ) def run(self) -> int: """ @@ -470,10 +469,11 @@ def run(self) -> int: self.print_stderr('Error: Invalid format specified.') return PolicyStatus.ERROR.value # Format and output data - handle empty results gracefully - data = formatter(self._sort_project_violations(dt_project_violations)) - self.print_to_file_or_stdout(data['details'], self.output) - self.print_to_file_or_stderr(data['summary'], self.status) + policy_output = formatter(self._sort_project_violations(dt_project_violations)) + self.print_to_file_or_stdout(policy_output.details, self.output) + self.print_to_file_or_stderr(policy_output.summary, self.status) # Return appropriate status based on violation count if len(dt_project_violations) > 0: return PolicyStatus.POLICY_FAIL.value return PolicyStatus.POLICY_SUCCESS.value + diff --git a/src/scanoss/inspection/policy_check.py b/src/scanoss/inspection/policy_check/policy_check.py similarity index 86% rename from src/scanoss/inspection/policy_check.py rename to src/scanoss/inspection/policy_check/policy_check.py index cd01972f..89263a30 100644 --- a/src/scanoss/inspection/policy_check.py +++ b/src/scanoss/inspection/policy_check/policy_check.py @@ -22,12 +22,12 @@ THE SOFTWARE. """ -from abc import abstractmethod +from abc import ABC, abstractmethod from enum import Enum -from typing import Any, Callable, Dict, Generic, List, TypeVar +from typing import Callable, Dict, Generic, List, NamedTuple, TypeVar -from ..scanossbase import ScanossBase -from .utils.license_utils import LicenseUtil +from ...scanossbase import ScanossBase +from ..utils.license_utils import LicenseUtil class PolicyStatus(Enum): @@ -46,9 +46,13 @@ class PolicyStatus(Enum): # End of PolicyStatus Class # +class PolicyOutput(NamedTuple): + details: str + summary: str + T = TypeVar('T') -class PolicyCheck(ScanossBase, Generic[T]): +class PolicyCheck(ScanossBase, Generic[T], ABC): """ A base class for implementing various software policy checks. @@ -80,7 +84,7 @@ def __init__( # noqa: PLR0913 self.output = output @abstractmethod - def run(self): + def run(self)-> tuple[int,PolicyOutput]: """ Execute the policy check process. @@ -91,14 +95,14 @@ def run(self): 3. Formatting the results 4. Saving the output to files if required - :return: A tuple containing: + :return: A named tuple containing two elements: - First element: PolicyStatus enum value (SUCCESS, FAIL, or ERROR) - - Second element: Dictionary containing the inspection results + - Second element: PolicyOutput A tuple containing the policy results. """ pass @abstractmethod - def _json(self, data: list[T]) -> Dict[str, Any]: + def _json(self, data: list[T]) -> PolicyOutput: """ Format the policy checks results as JSON. This method should be implemented by subclasses to create a Markdown representation @@ -112,7 +116,7 @@ def _json(self, data: list[T]) -> Dict[str, Any]: pass @abstractmethod - def _markdown(self, data: list[T]) -> Dict[str, Any]: + def _markdown(self, data: list[T]) -> PolicyOutput: """ Generate Markdown output for the policy check results. @@ -125,7 +129,7 @@ def _markdown(self, data: list[T]) -> Dict[str, Any]: pass @abstractmethod - def _jira_markdown(self, data: list[T]) -> Dict[str, Any]: + def _jira_markdown(self, data: list[T]) -> PolicyOutput: """ Generate Markdown output for the policy check results. @@ -137,7 +141,7 @@ def _jira_markdown(self, data: list[T]) -> Dict[str, Any]: """ pass - def _get_formatter(self) -> Callable[[List[dict]], Dict[str, Any]] or None: + def _get_formatter(self) -> Callable[[List[dict]], PolicyOutput]: """ Get the appropriate formatter function based on the specified format. @@ -145,7 +149,7 @@ def _get_formatter(self) -> Callable[[List[dict]], Dict[str, Any]] or None: """ valid_format = self._is_valid_format() if not valid_format: - return None + raise ValueError('Invalid format specified') # a map of which format function to return function_map = { 'json': self._json, @@ -205,14 +209,14 @@ def _generate_formatter_report(self, components: list[Dict]): if formatter is None: return PolicyStatus.ERROR.value, {} # Format the results - data = formatter(components) + policy_output = formatter(components) ## Save outputs if required - self.print_to_file_or_stdout(data['details'], self.output) - self.print_to_file_or_stderr(data['summary'], self.status) + self.print_to_file_or_stdout(policy_output.details, self.output) + self.print_to_file_or_stderr(policy_output.summary, self.status) # Check to see if we have policy violations if len(components) > 0: - return PolicyStatus.POLICY_FAIL.value, data - return PolicyStatus.POLICY_SUCCESS.value, data + return PolicyStatus.POLICY_FAIL.value, policy_output + return PolicyStatus.POLICY_SUCCESS.value, policy_output # # End of PolicyCheck Class # \ No newline at end of file diff --git a/src/scanoss/inspection/policy_check/scanoss/__init__.py b/src/scanoss/inspection/policy_check/scanoss/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/scanoss/inspection/raw/copyleft.py b/src/scanoss/inspection/policy_check/scanoss/copyleft.py similarity index 80% rename from src/scanoss/inspection/raw/copyleft.py rename to src/scanoss/inspection/policy_check/scanoss/copyleft.py index 97c25bab..08694854 100644 --- a/src/scanoss/inspection/raw/copyleft.py +++ b/src/scanoss/inspection/policy_check/scanoss/copyleft.py @@ -24,11 +24,11 @@ import json from dataclasses import dataclass -from typing import Any, Dict, List +from typing import Dict, List -from ..policy_check import PolicyStatus -from ..utils.markdown_utils import generate_jira_table, generate_table -from .raw_base import RawBase +from ...policy_check.policy_check import PolicyCheck, PolicyOutput, PolicyStatus +from ...utils.markdown_utils import generate_jira_table, generate_table +from ...utils.scan_result_processor import ScanResultProcessor @dataclass @@ -45,7 +45,7 @@ class Component: licenses: List[License] status: str -class Copyleft(RawBase[Component]): +class Copyleft(PolicyCheck[Component]): """ SCANOSS Copyleft class Inspects components for copyleft licenses @@ -78,17 +78,23 @@ def __init__( # noqa: PLR0913 :param exclude: Licenses to exclude from the analysis :param explicit: Explicitly defined licenses """ - super().__init__(debug, trace, quiet, format_type,filepath, output ,status, name='Copyleft Policy') + super().__init__( + debug, trace, quiet, format_type, status, name='Copyleft Policy', output=output + ) self.license_util.init(include, exclude, explicit) self.filepath = filepath - self.format = format self.output = output self.status = status - self.include = include - self.exclude = exclude - self.explicit = explicit + self.results_processor = ScanResultProcessor( + self.debug, + self.trace, + self.quiet, + self.filepath, + include, + exclude, + explicit) - def _json(self, components: list[Component]) -> Dict[str, Any]: + def _json(self, components: list[Component]) -> PolicyOutput: """ Format the components with copyleft licenses as JSON. @@ -96,16 +102,16 @@ def _json(self, components: list[Component]) -> Dict[str, Any]: :return: Dictionary with formatted JSON details and summary """ # A component is considered unique by its combination of PURL (Package URL) and license - component_licenses = self._group_components_by_license(components) + component_licenses = self.results_processor.group_components_by_license(components) details = {} if len(components) > 0: details = {'components': components} - return { - 'details': f'{json.dumps(details, indent=2)}\n', - 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', - } + return PolicyOutput( + details= f'{json.dumps(details, indent=2)}\n', + summary= f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', + ) - def _markdown(self, components: list[Component]) -> Dict[str, Any]: + def _markdown(self, components: list[Component]) -> PolicyOutput: """ Format the components with copyleft licenses as Markdown. @@ -114,7 +120,7 @@ def _markdown(self, components: list[Component]) -> Dict[str, Any]: """ return self._md_summary_generator(components, generate_table) - def _jira_markdown(self, components: list[Component]) -> Dict[str, Any]: + def _jira_markdown(self, components: list[Component]) -> PolicyOutput: """ Format the components with copyleft licenses as Markdown. @@ -123,7 +129,7 @@ def _jira_markdown(self, components: list[Component]) -> Dict[str, Any]: """ return self._md_summary_generator(components, generate_jira_table) - def _md_summary_generator(self, components: list[Component], table_generator): + def _md_summary_generator(self, components: list[Component], table_generator) -> PolicyOutput: """ Generates a Markdown summary for components with a focus on copyleft licenses. @@ -138,15 +144,10 @@ def _md_summary_generator(self, components: list[Component], table_generator): A callable function to generate tabular data for components. Returns: - dict - A dictionary containing two keys: - - 'details': A detailed Markdown representation including a table of components - and associated copyleft license data. - - 'summary': A textual summary highlighting the total number of components - with copyleft licenses. + PolicyOutput """ # A component is considered unique by its combination of PURL (Package URL) and license - component_licenses = self._group_components_by_license(components) + component_licenses = self.results_processor.group_components_by_license(components) headers = ['Component', 'License', 'URL', 'Copyleft'] centered_columns = [1, 4] rows = [] @@ -160,10 +161,10 @@ def _md_summary_generator(self, components: list[Component], table_generator): rows.append(row) # End license loop # End component loop - return { - 'details': f'### Copyleft Licenses\n{table_generator(headers, rows, centered_columns)}', - 'summary': f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', - } + return PolicyOutput( + details= f'### Copyleft Licenses\n{table_generator(headers, rows, centered_columns)}', + summary= f'{len(component_licenses)} component(s) with copyleft licenses were found.\n', + ) def _get_components_with_copyleft_licenses(self, components: list) -> list[Dict]: """ @@ -202,14 +203,13 @@ def _get_components(self): :return: A list of processed components with license data, or `None` if `self.results` is not set. """ - if self.results is None: + if self.results_processor.get_results() is None: return None - components: dict = {} # Extract component and license data from file and dependency results. Both helpers mutate `components` - self._get_components_data(self.results, components) - self._get_dependencies_data(self.results, components) - return self._convert_components_to_list(components) + self.results_processor.get_components_data(components) + self.results_processor.get_dependencies_data(components) + return self.results_processor.convert_components_to_list(components) def run(self): """ diff --git a/src/scanoss/inspection/raw/undeclared_component.py b/src/scanoss/inspection/policy_check/scanoss/undeclared_component.py similarity index 85% rename from src/scanoss/inspection/raw/undeclared_component.py rename to src/scanoss/inspection/policy_check/scanoss/undeclared_component.py index a7e32dac..ce122a5b 100644 --- a/src/scanoss/inspection/raw/undeclared_component.py +++ b/src/scanoss/inspection/policy_check/scanoss/undeclared_component.py @@ -24,11 +24,11 @@ import json from dataclasses import dataclass -from typing import Any, Dict, List +from typing import List -from ..policy_check import PolicyStatus -from ..utils.markdown_utils import generate_jira_table, generate_table -from .raw_base import RawBase +from ...policy_check.policy_check import PolicyCheck, PolicyOutput, PolicyStatus +from ...utils.markdown_utils import generate_jira_table, generate_table +from ...utils.scan_result_processor import ScanResultProcessor @dataclass @@ -44,7 +44,7 @@ class Component: licenses: List[License] status: str -class UndeclaredComponent(RawBase[Component]): +class UndeclaredComponent(PolicyCheck[Component]): """ SCANOSS UndeclaredComponent class Inspects for undeclared components @@ -59,7 +59,7 @@ def __init__( # noqa: PLR0913 format_type: str = 'json', status: str = None, output: str = None, - sbom_format: str = 'settings', + sbom_format: str = 'settings' ): """ Initialize the UndeclaredComponent class. @@ -74,13 +74,14 @@ def __init__( # noqa: PLR0913 :param sbom_format: Sbom format for status output (default 'settings') """ super().__init__( - debug, trace, quiet,format_type, filepath, output, status, name='Undeclared Components Policy' + debug, trace, quiet, format_type, status, name='Undeclared Components Policy', output=output ) self.filepath = filepath - self.format = format self.output = output self.status = status self.sbom_format = sbom_format + self.results_processor = ScanResultProcessor(self.debug, self.trace, self.quiet, self.filepath) + def _get_undeclared_components(self, components: list[Component]) -> list or None: """ @@ -163,7 +164,7 @@ def _get_summary(self, components: list) -> str: return summary - def _json(self, components: list[Component]) -> Dict[str, Any]: + def _json(self, components: list[Component]) -> PolicyOutput: """ Format the undeclared components as JSON. @@ -171,16 +172,16 @@ def _json(self, components: list[Component]) -> Dict[str, Any]: :return: Dictionary with formatted JSON details and summary """ # Use component grouped by licenses to generate the summary - component_licenses = self._group_components_by_license(components) + component_licenses = self.results_processor.group_components_by_license(components) details = {} if len(components) > 0: details = {'components': components} - return { - 'details': f'{json.dumps(details, indent=2)}\n', - 'summary': self._get_summary(component_licenses), - } + return PolicyOutput( + details=f'{json.dumps(details, indent=2)}\n', + summary=self._get_summary(component_licenses) + ) - def _markdown(self, components: list[Component]) -> Dict[str, Any]: + def _markdown(self, components: list[Component]) -> PolicyOutput: """ Format the undeclared components as Markdown. @@ -190,15 +191,15 @@ def _markdown(self, components: list[Component]) -> Dict[str, Any]: headers = ['Component', 'License'] rows = [] # TODO look at using SpdxLite license name lookup method - component_licenses = self._group_components_by_license(components) + component_licenses = self.results_processor.group_components_by_license(components) for component in component_licenses: rows.append([component.get('purl'), component.get('spdxid')]) - return { - 'details': f'### Undeclared components\n{generate_table(headers, rows)}\n', - 'summary': self._get_summary(component_licenses), - } + return PolicyOutput( + details= f'### Undeclared components\n{generate_table(headers, rows)}\n', + summary= self._get_summary(component_licenses), + ) - def _jira_markdown(self, components: list) -> Dict[str, Any]: + def _jira_markdown(self, components: list) -> PolicyOutput: """ Format the undeclared components as Markdown. @@ -208,13 +209,13 @@ def _jira_markdown(self, components: list) -> Dict[str, Any]: headers = ['Component', 'License'] rows = [] # TODO look at using SpdxLite license name lookup method - component_licenses = self._group_components_by_license(components) + component_licenses = self.results_processor.group_components_by_license(components) for component in component_licenses: rows.append([component.get('purl'), component.get('spdxid')]) - return { - 'details': f'{generate_jira_table(headers, rows)}', - 'summary': self._get_jira_summary(component_licenses), - } + return PolicyOutput( + details= f'{generate_jira_table(headers, rows)}', + summary= self._get_jira_summary(component_licenses), + ) def _get_unique_components(self, components: list) -> list: """ @@ -272,13 +273,13 @@ def _get_components(self): :return: A list of processed components with their licenses, or `None` if `self.results` is not set. """ - if self.results is None: + if self.results_processor.get_results() is None: return None components: dict = {} # Extract file and snippet components - components = self._get_components_data(self.results, components) + components = self.results_processor.get_components_data(components) # Convert to list and process licenses - return self._convert_components_to_list(components) + return self.results_processor.convert_components_to_list(components) def run(self): """ diff --git a/src/scanoss/inspection/summary/__init__.py b/src/scanoss/inspection/summary/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/scanoss/inspection/raw/component_summary.py b/src/scanoss/inspection/summary/component_summary.py similarity index 81% rename from src/scanoss/inspection/raw/component_summary.py rename to src/scanoss/inspection/summary/component_summary.py index 03bddbd8..b4563176 100644 --- a/src/scanoss/inspection/raw/component_summary.py +++ b/src/scanoss/inspection/summary/component_summary.py @@ -24,11 +24,36 @@ import json from typing import Any -from ..policy_check import T -from .raw_base import RawBase +from ...scanossbase import ScanossBase +from ..policy_check.policy_check import T +from ..utils.scan_result_processor import ScanResultProcessor + + +class ComponentSummary(ScanossBase): + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + filepath: str = None, + format_type: str = 'json', + output: str = None, + ): + """ + Initialize the ComponentSummary class. + :param debug: Enable debug mode + :param trace: Enable trace mode + :param quiet: Enable quiet mode + :param filepath: Path to the file containing component data + :param format_type: Output format ('json' or 'md') + """ + super().__init__(debug, trace, quiet) + self.filepath = filepath + self.output = output + self.results_processor = ScanResultProcessor(debug, trace, quiet, filepath) -class ComponentSummary(RawBase): def _json(self, data: dict[str,Any]) -> dict[str,Any]: """ @@ -77,11 +102,11 @@ def _get_component_summary_from_components(self, scan_components: list)-> dict: """ Get a component summary from detected components. - :param components: List of all components + :param scan_components: List of all components :return: Dict with license summary information """ # A component is considered unique by its combination of PURL (Package URL) and license - component_licenses = self._group_components_by_license(scan_components) + component_licenses = self.results_processor.group_components_by_license(scan_components) total_components = len(component_licenses) # Get undeclared components undeclared_components = len([c for c in component_licenses if c['status'] == 'pending']) @@ -121,13 +146,13 @@ def _get_components(self): :return: A list of processed components with license data, or `None` if `self.results` is not set. """ - if self.results is None: - raise ValueError(f'Error: No results found in ${self.filepath}') + if self.results_processor.get_results() is None: + raise ValueError(f'Error: No results found in {self.filepath}') components: dict = {} # Extract component and license data from file and dependency results. Both helpers mutate `components` - self._get_components_data(self.results, components) - return self._convert_components_to_list(components) + self.results_processor.get_components_data(components) + return self.results_processor.convert_components_to_list(components) def _format(self, component_summary) -> str: # TODO: Implement formatter to support dynamic outputs diff --git a/src/scanoss/inspection/raw/license_summary.py b/src/scanoss/inspection/summary/license_summary.py similarity index 89% rename from src/scanoss/inspection/raw/license_summary.py rename to src/scanoss/inspection/summary/license_summary.py index c849ea0a..3cf3761e 100644 --- a/src/scanoss/inspection/raw/license_summary.py +++ b/src/scanoss/inspection/summary/license_summary.py @@ -25,11 +25,12 @@ import json from typing import Any -from ..policy_check import T -from .raw_base import RawBase +from ...scanossbase import ScanossBase +from ..policy_check.policy_check import T +from ..utils.scan_result_processor import ScanResultProcessor -class LicenseSummary(RawBase): +class LicenseSummary(ScanossBase): """ SCANOSS LicenseSummary class Inspects results and generates comprehensive license summaries from detected components. @@ -38,6 +39,42 @@ class LicenseSummary(RawBase): information, providing detailed summaries including copyleft analysis and license statistics. """ + # Define required license fields as class constants + REQUIRED_LICENSE_FIELDS = ['spdxid', 'url', 'copyleft', 'source'] + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + filepath: str = None, + status: str = None, + output: str = None, + include: str = None, + exclude: str = None, + explicit: str = None, + ): + """ + Initialize the LicenseSummary class. + + :param debug: Enable debug mode + :param trace: Enable trace mode + :param quiet: Enable quiet mode + :param filepath: Path to the file containing component data + :param output: Path to save detailed output + :param include: Licenses to include in the analysis + :param exclude: Licenses to exclude from the analysis + :param explicit: Explicitly defined licenses + """ + super().__init__(debug=debug, trace=trace, quiet=quiet) + self.results_processor = ScanResultProcessor(debug, trace, quiet, filepath, include, exclude, explicit) + self.filepath = filepath + self.output = output + self.status = status + self.include = include + self.exclude = exclude + self.explicit = explicit + def _json(self, data: dict[str,Any]) -> dict[str, Any]: """ Format license summary data as JSON. @@ -78,41 +115,6 @@ def _jira_markdown(self, data: list[T]) -> dict[str, Any]: """ pass - # Define required license fields as class constants - REQUIRED_LICENSE_FIELDS = ['spdxid', 'url', 'copyleft', 'source'] - - def __init__( # noqa: PLR0913 - self, - debug: bool = False, - trace: bool = False, - quiet: bool = False, - filepath: str = None, - status: str = None, - output: str = None, - include: str = None, - exclude: str = None, - explicit: str = None, - ): - """ - Initialize the LicenseSummary class. - - :param debug: Enable debug mode - :param trace: Enable trace mode (default True) - :param quiet: Enable quiet mode - :param filepath: Path to the file containing component data - :param output: Path to save detailed output - :param include: Licenses to include in the analysis - :param exclude: Licenses to exclude from the analysis - :param explicit: Explicitly defined licenses - """ - super().__init__(debug, trace, quiet, filepath = filepath, output=output) - self.license_util.init(include, exclude, explicit) - self.filepath = filepath - self.output = output - self.status = status - self.include = include - self.exclude = exclude - self.explicit = explicit def _get_licenses_summary_from_components(self, components: list)-> dict: """ @@ -122,7 +124,7 @@ def _get_licenses_summary_from_components(self, components: list)-> dict: :return: Dict with license summary information """ # A component is considered unique by its combination of PURL (Package URL) and license - component_licenses = self._group_components_by_license(components) + component_licenses = self.results_processor.group_components_by_license(components) license_component_count = {} # Count license per component for lic in component_licenses: @@ -164,14 +166,14 @@ def _get_components(self): :return: A list of processed components with license data, or `None` if `self.results` is not set. """ - if self.results is None: - raise ValueError(f'Error: No results found in ${self.filepath}') + if self.results_processor.get_results() is None: + raise ValueError(f'Error: No results found in {self.filepath}') components: dict = {} # Extract component and license data from file and dependency results. Both helpers mutate `components` - self._get_components_data(self.results, components) - self._get_dependencies_data(self.results, components) - return self._convert_components_to_list(components) + self.results_processor.get_components_data(components) + self.results_processor.get_dependencies_data(components) + return self.results_processor.convert_components_to_list(components) def _format(self, license_summary) -> str: # TODO: Implement formatter to support dynamic outputs diff --git a/src/scanoss/inspection/raw/match_summary.py b/src/scanoss/inspection/summary/match_summary.py similarity index 82% rename from src/scanoss/inspection/raw/match_summary.py rename to src/scanoss/inspection/summary/match_summary.py index 645824bb..b79233e5 100644 --- a/src/scanoss/inspection/raw/match_summary.py +++ b/src/scanoss/inspection/summary/match_summary.py @@ -94,6 +94,7 @@ def __init__( # noqa: PLR0913 self.scanoss_results_path = scanoss_results_path self.line_range_prefix = line_range_prefix self.output = output + self.print_debug("Initializing MatchSummary class") def _get_match_summary_item(self, file_name: str, result: dict) -> MatchSummaryItem: @@ -108,11 +109,16 @@ def _get_match_summary_item(self, file_name: str, result: dict) -> MatchSummaryI :param result: SCANOSS scan result dictionary containing match details :return: Populated match summary item with all relevant information """ + self.print_trace(f"Creating match summary item for file: {file_name}, id: {result.get('id')}") + if result.get('id') == "snippet": # Snippet match: create URL with line range anchor lines = scanoss_scan_results_utils.get_lines(result.get('lines')) end_line = lines[len(lines) - 1] if len(lines) > 1 else lines[0] file_url = f"{self.line_range_prefix}/{file_name}#L{lines[0]}-L{end_line}" + + self.print_trace(f"Snippet match: lines {lines[0]}-{end_line}, purl: {result.get('purl')[0]}") + return MatchSummaryItem( file_url=file_url, file=file_name, @@ -124,6 +130,8 @@ def _get_match_summary_item(self, file_name: str, result: dict) -> MatchSummaryI lines=f"{lines[0]}-{lines[len(lines) - 1] if len(lines) > 1 else lines[0]}" ) # File match: create URL without line range + self.print_trace(f"File match: {file_name}, purl: {result.get('purl')[0]}, version: {result.get('version')}") + return MatchSummaryItem( file=file_name, file_url=f"{self.line_range_prefix}/{file_name}", @@ -176,12 +184,19 @@ def _get_matches_summary(self) -> ComponentMatchSummary: required fields and categorizing matches into file matches and snippet matches. Skips invalid or incomplete results with debug messages. """ + self.print_debug(f"Loading scan results from: {self.scanoss_results_path}") + # Load scan results from JSON file scan_results = load_json_file(self.scanoss_results_path) gitlab_matches_summary = ComponentMatchSummary(files=[], snippet=[]) + self.print_debug(f"Processing {len(scan_results)} files from scan results") + self.print_trace(f"Line range prefix set to: {self.line_range_prefix}") + # Process each file and its results for file_name, results in scan_results.items(): + self.print_trace(f"Processing file: {file_name} with {len(results)} results") + for result in results: # Skip non-matches if result.get('id') == "none": @@ -196,8 +211,15 @@ def _get_matches_summary(self) -> ComponentMatchSummary: summary_item = self._get_match_summary_item(file_name, result) if result.get('id') == "snippet": gitlab_matches_summary.snippet.append(summary_item) + self.print_trace(f"Added snippet match for {file_name}") else: gitlab_matches_summary.files.append(summary_item) + self.print_trace(f"Added file match for {file_name}") + + self.print_debug( + f"Match summary complete: {len(gitlab_matches_summary.files)} file matches, " + f"{len(gitlab_matches_summary.snippet)} snippet matches" + ) return gitlab_matches_summary @@ -212,14 +234,23 @@ def _markdown(self, gitlab_matches_summary: ComponentMatchSummary) -> str: :param gitlab_matches_summary: Container with categorized file and snippet matches to format :return: Complete Markdown document with formatted match tables """ + self.print_debug("Generating Markdown from match summaries") if len(gitlab_matches_summary.files) == 0 and len(gitlab_matches_summary.snippet) == 0: + self.print_debug("No matches to format - returning empty string") return "" + self.print_trace( + f"Formatting {len(gitlab_matches_summary.files)} file matches and " + f"{len(gitlab_matches_summary.snippet)} snippet matches" + ) + # Define table headers file_match_headers = ['File', 'License', 'Similarity', 'PURL', 'Version'] snippet_match_headers = ['File', 'License', 'Similarity', 'PURL', 'Version', 'Lines'] + # Build file matches table + self.print_trace("Building file matches table") file_match_rows = [] for file_match in gitlab_matches_summary.files: row = [ @@ -233,6 +264,7 @@ def _markdown(self, gitlab_matches_summary: ComponentMatchSummary) -> str: file_match_table = generate_table(file_match_headers, file_match_rows) # Build snippet matches table + self.print_trace("Building snippet matches table") snippet_match_rows = [] for snippet_match in gitlab_matches_summary.snippet: row = [ @@ -262,6 +294,8 @@ def _markdown(self, gitlab_matches_summary: ComponentMatchSummary) -> str: markdown += snippet_match_table markdown += "\n\n" + self.print_trace(f"Markdown generation complete (length: {len(markdown)} characters)") + self.print_debug("Match summary Markdown generation complete") return markdown def run(self): @@ -275,16 +309,33 @@ def run(self): 3. Generates Markdown report 4. Outputs to file or stdout """ + self.print_debug("Starting match summary generation process") + self.print_trace( + f"Configuration - Results path: {self.scanoss_results_path}, Output: {self.output}, " + f"Line range prefix: {self.line_range_prefix}" + ) + # Load and process scan results into categorized matches + self.print_trace("Loading and processing scan results") matches = self._get_matches_summary() # Format matches as GitLab-compatible Markdown + self.print_trace("Generating Markdown output") matches_md = self._markdown(matches) if matches_md == "": + self.print_debug("No matches found - exiting") self.print_stdout("No matches found.") return + # Output to file or stdout + self.print_trace("Writing output") + if self.output: + self.print_debug(f"Writing match summary to file: {self.output}") + else: + self.print_debug("Writing match summary to 'stdout'") + self.print_to_file_or_stdout(matches_md, self.output) + self.print_debug("Match summary generation complete") diff --git a/src/scanoss/inspection/raw/raw_base.py b/src/scanoss/inspection/utils/scan_result_processor.py similarity index 88% rename from src/scanoss/inspection/raw/raw_base.py rename to src/scanoss/inspection/utils/scan_result_processor.py index 8a1a6f8d..22333b5d 100644 --- a/src/scanoss/inspection/raw/raw_base.py +++ b/src/scanoss/inspection/utils/scan_result_processor.py @@ -22,11 +22,10 @@ THE SOFTWARE. """ -from abc import abstractmethod from enum import Enum from typing import Any, Dict, TypeVar -from ..policy_check import PolicyCheck +from ...scanossbase import ScanossBase from ..utils.file_utils import load_json_file from ..utils.license_utils import LicenseUtil @@ -51,12 +50,13 @@ class ComponentID(Enum): # T = TypeVar('T') -class RawBase(PolicyCheck[T]): +class ScanResultProcessor(ScanossBase): """ - A base class to perform inspections over scan results. + A utility class for processing and transforming scan results. - This class provides a basic for scan results inspection, including methods for - processing scan results components and licenses. + This class provides functionality for processing scan results, including methods for + loading, parsing, extracting, and aggregating component and license data from scan results. + It serves as a shared data processing layer used by both policy checks and summary generators. Inherits from: ScanossBase: A base class providing common functionality for SCANOSS-related operations. @@ -67,40 +67,19 @@ def __init__( # noqa: PLR0913 debug: bool = False, trace: bool = False, quiet: bool = False, - format_type: str = None, - filepath: str = None, - output: str = None, - status: str = None, - name: str = None, + result_file_path: str = None, + include: str = None, + exclude: str = None, + explicit: str = None, ): - super().__init__(debug, trace, quiet, format_type,status, name, output) + super().__init__(debug, trace, quiet) + self.result_file_path = result_file_path self.license_util = LicenseUtil() - self.filepath = filepath - self.output = output + self.license_util.init(include, exclude, explicit) self.results = self._load_input_file() - @abstractmethod - def _get_components(self): - """ - Retrieve and process components from the preloaded results. - - This method performs the following steps: - 1. Checks if the results have been previously loaded (self.results). - 2. Extracts and processes components from the loaded results. - - :return: A list of processed components, or None if an error occurred during any step. - - Possible reasons for returning None include: - - Results not loaded (self.results is None) - - Failure to extract components from the results - - Note: - - This method assumes that the results have been previously loaded and stored in self.results. - - Implementations must extract components (e.g. via `_get_components_data`, - `_get_dependencies_data`, or other helpers). - - If `self.results` is `None`, simply return `None`. - """ - pass + def get_results(self) -> Dict[str, Any]: + return self.results def _append_component(self, components: Dict[str, Any], new_component: Dict[str, Any]) -> Dict[str, Any]: """ @@ -213,7 +192,7 @@ def _update_component_counters(self, component, status): else: component['undeclared'] += 1 - def _get_components_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: + def get_components_data(self, components: Dict[str, Any]) -> Dict[str, Any]: """ Extract and process file and snippet components from results. @@ -230,11 +209,11 @@ def _get_components_data(self, results: Dict[str, Any], components: Dict[str, An which tracks the number of occurrences of each license Args: - results: A dictionary containing the raw results of a component scan + components: A dictionary containing the raw results of a component scan Returns: Updated components dictionary with file and snippet data """ - for component in results.values(): + for component in self.results.values(): for c in component: component_id = c.get('id') if not component_id: @@ -266,15 +245,13 @@ def _get_components_data(self, results: Dict[str, Any], components: Dict[str, An # End components loop return components - def _get_dependencies_data(self, results: Dict[str, Any], components: Dict[str, Any]) -> Dict[str, Any]: + def get_dependencies_data(self,components: Dict[str, Any]) -> Dict[str, Any]: """ Extract and process dependency components from results. - - :param results: A dictionary containing the raw results of a component scan :param components: Existing components dictionary to update :return: Updated components dictionary with dependency data """ - for component in results.values(): + for component in self.results.values(): for c in component: component_id = c.get('id') if not component_id: @@ -313,12 +290,12 @@ def _load_input_file(self): Dict[str, Any]: The parsed JSON data """ try: - return load_json_file(self.filepath) + return load_json_file(self.result_file_path) except Exception as e: self.print_stderr(f'ERROR: Problem parsing input JSON: {e}') return None - def _convert_components_to_list(self, components: dict): + def convert_components_to_list(self, components: dict): if components is None: self.print_debug(f'WARNING: Components is empty {self.results}') return None @@ -372,7 +349,7 @@ def _get_licenses_order_by_source_priority(self,licenses_data): self.print_debug("No priority sources found, returning all licenses as list") return licenses_data - def _group_components_by_license(self,components): + def group_components_by_license(self,components): """ Groups components by their unique component-license pairs. @@ -425,5 +402,5 @@ def _group_components_by_license(self,components): # -# End of PolicyCheck Class -# +# End of ScanResultProcessor Class +# \ No newline at end of file diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 24db1ab7..0ccf7886 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -28,12 +28,14 @@ import unittest from unittest.mock import Mock, patch -from src.scanoss.inspection.policy_check import PolicyStatus -from src.scanoss.inspection.raw.component_summary import ComponentSummary -from src.scanoss.inspection.raw.copyleft import Copyleft -from src.scanoss.inspection.raw.license_summary import LicenseSummary -from src.scanoss.inspection.raw.undeclared_component import UndeclaredComponent -from src.scanoss.inspection.dependency_track.project_violation import DependencyTrackProjectViolationPolicyCheck +from src.scanoss.inspection.policy_check.dependency_track.project_violation import ( + DependencyTrackProjectViolationPolicyCheck, +) +from src.scanoss.inspection.policy_check.policy_check import PolicyStatus +from src.scanoss.inspection.policy_check.scanoss.copyleft import Copyleft +from src.scanoss.inspection.policy_check.scanoss.undeclared_component import UndeclaredComponent +from src.scanoss.inspection.summary.component_summary import ComponentSummary +from src.scanoss.inspection.summary.license_summary import LicenseSummary class MyTestCase(unittest.TestCase): @@ -67,11 +69,11 @@ def test_empty_copyleft_policy(self): file_name = 'result-no-copyleft.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json') - status, data = copyleft.run() - details = json.loads(data['details']) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) self.assertEqual(status, PolicyStatus.POLICY_SUCCESS.value) self.assertEqual(details, {}) - self.assertEqual(data['summary'], '0 component(s) with copyleft licenses were found.\n') + self.assertEqual(policy_output.summary, '0 component(s) with copyleft licenses were found.\n') """ Inspect for copyleft licenses include @@ -82,9 +84,9 @@ def test_copyleft_policy_include(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', include='MIT') - status, data = copyleft.run() + status, policy_output = copyleft.run() has_mit_license = False - details = json.loads(data['details']) + details = json.loads(policy_output.details) for component in details['components']: for license in component['licenses']: if license['spdxid'] == 'MIT': @@ -103,8 +105,8 @@ def test_copyleft_policy_exclude(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', exclude='GPL-2.0-only') - status, data = copyleft.run() - results = json.loads(data['details']) + status, policy_output = copyleft.run() + results = json.loads(policy_output.details) self.assertEqual(results, {}) self.assertEqual(status, PolicyStatus.POLICY_SUCCESS.value) @@ -117,8 +119,8 @@ def test_copyleft_policy_explicit(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='MIT') - status, data = copyleft.run() - results = json.loads(data['details']) + status, policy_output = copyleft.run() + results = json.loads(policy_output.details) self.assertEqual(len(results['components']), 2) self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) @@ -131,8 +133,8 @@ def test_copyleft_policy_empty_explicit(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='json', explicit='') - status, data = copyleft.run() - results = json.loads(data['details']) + status, policy_output = copyleft.run() + results = json.loads(policy_output.details) self.assertEqual(len(results['components']), 5) self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) @@ -145,7 +147,7 @@ def test_copyleft_policy_markdown(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='md', explicit='MIT') - status, data = copyleft.run() + status, policy_output = copyleft.run() expected_detail_output = ( '### Copyleft Licenses \n | Component | License | URL | Copyleft |\n' ' | - | :-: | - | - |\n' @@ -154,10 +156,10 @@ def test_copyleft_policy_markdown(self): ) expected_summary_output = '2 component(s) with copyleft licenses were found.\n' self.assertEqual( - re.sub(r'\s|\\(?!`)|\\(?=`)', '', data['details']), + re.sub(r'\s|\\(?!`)|\\(?=`)', '', policy_output.details), re.sub(r'\s|\\(?!`)|\\(?=`)', '', expected_detail_output), ) - self.assertEqual(data['summary'], expected_summary_output) + self.assertEqual(policy_output.summary, expected_summary_output) self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) ## Undeclared Components Policy Tests ## @@ -180,9 +182,9 @@ def test_undeclared_policy(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='json', sbom_format='legacy') - status, data = undeclared.run() - results = json.loads(data['details']) - summary = data['summary'] + status, policy_output = undeclared.run() + results = json.loads(policy_output.details) + summary = policy_output.summary expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `sbom.json` file ```json @@ -215,9 +217,9 @@ def test_undeclared_policy_markdown(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md', sbom_format='legacy') - status, data = undeclared.run() - results = data['details'] - summary = data['summary'] + status, policy_output = undeclared.run() + results = policy_output.details + summary = policy_output.summary expected_details_output = """ ### Undeclared components | Component | License | | - | - | @@ -259,9 +261,9 @@ def test_undeclared_policy_markdown_scanoss_summary(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='md') - status, data = undeclared.run() - results = data['details'] - summary = data['summary'] + status, policy_output = undeclared.run() + results = policy_output.details + summary = policy_output.summary expected_details_output = """ ### Undeclared components | Component | License | | - | - | @@ -306,9 +308,9 @@ def test_undeclared_policy_scanoss_summary(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name) - status, data = undeclared.run() - results = json.loads(data['details']) - summary = data['summary'] + status, policy_output = undeclared.run() + results = json.loads(policy_output.details) + summary = policy_output.summary expected_summary_output = """3 undeclared component(s) were found. Add the following snippet into your `scanoss.json` file @@ -340,9 +342,9 @@ def test_undeclared_policy_jira_markdown_output(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) undeclared = UndeclaredComponent(filepath=input_file_name, format_type='jira_md') - status, data = undeclared.run() - details = data['details'] - summary = data['summary'] + status, policy_output = undeclared.run() + details = policy_output.details + summary = policy_output.summary expected_details_output = """|*Component*|*License*| |pkg:github/scanoss/jenkins-pipeline-example|unknown| |pkg:github/scanoss/scanner.c|GPL-2.0-only| @@ -377,8 +379,8 @@ def test_copyleft_policy_jira_markdown_output(self): file_name = 'result.json' input_file_name = os.path.join(script_dir, 'data', file_name) copyleft = Copyleft(filepath=input_file_name, format_type='jira_md') - status, data = copyleft.run() - results = data['details'] + status, policy_output = copyleft.run() + results = policy_output.details expected_details_output = """### Copyleft Licenses\n|*Component*|*License*|*URL*|*Copyleft*| |pkg:github/scanoss/scanner.c|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| |pkg:github/scanoss/engine|GPL-2.0-only|https://spdx.org/licenses/GPL-2.0-only.html|YES| @@ -436,7 +438,7 @@ def test_inspect_component_summary_empty_result(self): ## Dependency Track Project Violation Policy Tests ## - @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + @patch('src.scanoss.inspection.policy_check.dependency_track.project_violation.DependencyTrackService') def test_dependency_track_project_violation_json_formatter(self, mock_service): mock_service.return_value = Mock() project_violation = DependencyTrackProjectViolationPolicyCheck( @@ -464,14 +466,12 @@ def test_dependency_track_project_violation_json_formatter(self, mock_service): } ] result = project_violation._json(test_violations) - self.assertIn('details', result) - self.assertIn('summary', result) - self.assertEqual(result['summary'], '1 policy violations were found.\n') - details = json.loads(result['details']) + self.assertEqual(result.summary, '1 policy violations were found.\n') + details = json.loads(result.details) self.assertEqual(len(details), 1) self.assertEqual(details[0]['type'], 'SECURITY') - @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + @patch('src.scanoss.inspection.policy_check.dependency_track.project_violation.DependencyTrackService') def test_dependency_track_project_violation_markdown_formatter(self, mock_service): mock_service.return_value = Mock() project_violation = DependencyTrackProjectViolationPolicyCheck( @@ -499,16 +499,14 @@ def test_dependency_track_project_violation_markdown_formatter(self, mock_servic } ] result = project_violation._markdown(test_violations) - self.assertIn('details', result) - self.assertIn('summary', result) - self.assertEqual(result['summary'], '1 policy violations were found.\n') - self.assertIn('State', result['details']) - self.assertIn('Risk Type', result['details']) - self.assertIn('Policy Name', result['details']) - self.assertIn('Component', result['details']) - self.assertIn('Date', result['details']) - - @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + self.assertEqual(result.summary, '1 policy violations were found.\n') + self.assertIn('State', result.details) + self.assertIn('Risk Type', result.details) + self.assertIn('Policy Name', result.details) + self.assertIn('Component', result.details) + self.assertIn('Date', result.details) + + @patch('src.scanoss.inspection.policy_check.dependency_track.project_violation.DependencyTrackService') def test_dependency_track_project_violation_sort_violations(self, mock_service): mock_service.return_value = Mock() project_violation = DependencyTrackProjectViolationPolicyCheck( @@ -528,7 +526,7 @@ def test_dependency_track_project_violation_sort_violations(self, mock_service): self.assertEqual(sorted_violations[2]['type'], 'LICENSE') self.assertEqual(sorted_violations[3]['type'], 'OTHER') - @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + @patch('src.scanoss.inspection.policy_check.dependency_track.project_violation.DependencyTrackService') def test_dependency_track_project_violation_empty_violations(self, mock_service): mock_service.return_value = Mock() project_violation = DependencyTrackProjectViolationPolicyCheck( @@ -539,11 +537,11 @@ def test_dependency_track_project_violation_empty_violations(self, mock_service) ) empty_violations = [] result = project_violation._json(empty_violations) - self.assertEqual(result['summary'], '0 policy violations were found.\n') - details = json.loads(result['details']) + self.assertEqual(result.summary, '0 policy violations were found.\n') + details = json.loads(result.details) self.assertEqual(len(details), 0) - @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + @patch('src.scanoss.inspection.policy_check.dependency_track.project_violation.DependencyTrackService') def test_dependency_track_project_violation_markdown_empty(self, mock_service): mock_service.return_value = Mock() project_violation = DependencyTrackProjectViolationPolicyCheck( @@ -554,11 +552,11 @@ def test_dependency_track_project_violation_markdown_empty(self, mock_service): ) empty_violations = [] result = project_violation._markdown(empty_violations) - self.assertEqual(result['summary'], '0 policy violations were found.\n') - self.assertIn('State', result['details']) - self.assertIn('Risk Type', result['details']) + self.assertEqual(result.summary, '0 policy violations were found.\n') + self.assertIn('State', result.details) + self.assertIn('Risk Type', result.details) - @patch('src.scanoss.inspection.dependency_track.project_violation.DependencyTrackService') + @patch('src.scanoss.inspection.policy_check.dependency_track.project_violation.DependencyTrackService') def test_dependency_track_project_violation_multiple_types(self, mock_service): mock_service.return_value = Mock() project_violation = DependencyTrackProjectViolationPolicyCheck( @@ -602,8 +600,8 @@ def test_dependency_track_project_violation_multiple_types(self, mock_service): } ] result = project_violation._json(test_violations) - self.assertEqual(result['summary'], '2 policy violations were found.\n') - details = json.loads(result['details']) + self.assertEqual(result.summary, '2 policy violations were found.\n') + details = json.loads(result.details) self.assertEqual(len(details), 2) if __name__ == '__main__': diff --git a/tools/linter.sh b/tools/linter.sh new file mode 100755 index 00000000..04d1e76d --- /dev/null +++ b/tools/linter.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# +# Lint Python files changed since merge base with origin/main +# Usage: linter.sh [--fix] [--docker] + +set -e + +# Parse arguments +FIX_FLAG="" +USE_DOCKER=false + +while [[ $# -gt 0 ]]; do + case $1 in + --fix) + FIX_FLAG="--fix" + shift + ;; + --docker) + USE_DOCKER=true + shift + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [--fix] [--docker]" + exit 1 + ;; + esac +done + +# Find merge base with origin/main +merge_base=$(git merge-base origin/main HEAD) + +# Get all changed Python files since merge base +files=$(git diff --name-only "$merge_base" HEAD | grep '\.py$' || true) + +# Check if there are any Python files changed +if [ -z "$files" ]; then + echo "No Python files changed" + exit 0 +fi + +# Run linter +if [ "$USE_DOCKER" = true ]; then + # Run with Docker + docker run --rm -v "$(pwd)":/src -w /src ghcr.io/astral-sh/ruff:0.14.2 check ${files} ${FIX_FLAG} +else + # Run locally + python3 -m ruff check ${files} ${FIX_FLAG} +fi \ No newline at end of file From 47d6430fb4cfa756f6b7ede30759bffa8286f630 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 14 Nov 2025 17:42:00 +0100 Subject: [PATCH 402/489] fix(cli): terminal cursor disappears after aborting scan with ctrl+c --- CHANGELOG.md | 4 +- src/scanoss/filecount.py | 75 +++--- src/scanoss/scanner.py | 371 +++++++++++++------------- src/scanoss/scanners/folder_hasher.py | 48 ++-- src/scanoss/scanners/scanner_hfh.py | 35 +-- src/scanoss/threadedscanning.py | 10 + 6 files changed, 275 insertions(+), 268 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a19331d0..5cc984f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -### Added -- Upcoming changes... +### Fixed +- Fixed terminal cursor disappearing after aborting scan with Ctrl+C ## [1.40.1] - 2025-10-29 ### Changed diff --git a/src/scanoss/filecount.py b/src/scanoss/filecount.py index a2f43b1b..87b8df75 100644 --- a/src/scanoss/filecount.py +++ b/src/scanoss/filecount.py @@ -26,6 +26,7 @@ import os import pathlib import sys +from contextlib import nullcontext from progress.spinner import Spinner @@ -105,48 +106,46 @@ def count_files(self, scan_dir: str) -> bool: """ success = True if not scan_dir: - raise Exception(f'ERROR: Please specify a folder to scan') + raise Exception('ERROR: Please specify a folder to scan') if not os.path.exists(scan_dir) or not os.path.isdir(scan_dir): raise Exception(f'ERROR: Specified folder does not exist or is not a folder: {scan_dir}') self.print_msg(f'Searching {scan_dir} for files to count...') - spinner = None - if not self.quiet and self.isatty: - spinner = Spinner('Searching ') - file_types = {} - file_count = 0 - file_size = 0 - for root, dirs, files in os.walk(scan_dir): - self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') - dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories - filtered_files = self.__filter_files(files) # Strip out unwanted files - self.print_trace(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}') - for file in filtered_files: # Cycle through each filtered file - path = os.path.join(root, file) - f_size = 0 - try: - f_size = os.stat(path).st_size - except Exception as e: - self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # broken symlink - if f_size > 0: # Ignore broken links and empty files - file_count = file_count + 1 - file_size = file_size + f_size - f_suffix = pathlib.Path(file).suffix - if not f_suffix or f_suffix == '': - f_suffix = 'no_suffix' - self.print_trace(f'Counting {path} ({f_suffix} - {f_size})..') - fc = file_types.get(f_suffix) - if not fc: - fc = [1, f_size] - else: - fc[0] = fc[0] + 1 - fc[1] = fc[1] + f_size - file_types[f_suffix] = fc - if spinner: - spinner.next() - # End for loop - if spinner: - spinner.finish() + spinner_ctx = Spinner('Searching ') if (not self.quiet and self.isatty) else nullcontext() + + with spinner_ctx as spinner: + file_types = {} + file_count = 0 + file_size = 0 + for root, dirs, files in os.walk(scan_dir): + self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}') + dirs[:] = self.__filter_dirs(dirs) # Strip out unwanted directories + filtered_files = self.__filter_files(files) # Strip out unwanted files + self.print_trace(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}') + for file in filtered_files: # Cycle through each filtered file + path = os.path.join(root, file) + f_size = 0 + try: + f_size = os.stat(path).st_size + except Exception as e: + self.print_trace(f'Ignoring missing symlink file: {file} ({e})') # broken symlink + if f_size > 0: # Ignore broken links and empty files + file_count = file_count + 1 + file_size = file_size + f_size + f_suffix = pathlib.Path(file).suffix + if not f_suffix or f_suffix == '': + f_suffix = 'no_suffix' + self.print_trace(f'Counting {path} ({f_suffix} - {f_size})..') + fc = file_types.get(f_suffix) + if not fc: + fc = [1, f_size] + else: + fc[0] = fc[0] + 1 + fc[1] = fc[1] + f_size + file_types[f_suffix] = fc + if spinner: + spinner.next() + # End for loop self.print_stderr(f'Found {file_count:,.0f} files with a total size of {file_size / (1 << 20):,.2f} MB.') if file_types: csv_dict = [] diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 63803e54..6e5d147b 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -26,6 +26,7 @@ import json import os import sys +from contextlib import nullcontext from pathlib import Path from typing import Any, Dict, List, Optional @@ -363,62 +364,60 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 operation_type='scanning', ) self.print_msg(f'Searching {scan_dir} for files to fingerprint...') - spinner = None - if not self.quiet and self.isatty: - spinner = Spinner('Fingerprinting ') - save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan - wfp_list = [] - scan_block = '' - scan_size = 0 - queue_size = 0 - file_count = 0 # count all files fingerprinted - wfp_file_count = 0 # count number of files in each queue post - scan_started = False - - to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) - for to_scan_file in to_scan_files: - if self.threaded_scan and self.threaded_scan.stop_scanning(): - self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') - break - self.print_debug(f'Fingerprinting {to_scan_file}...') - if spinner: - spinner.next() - abs_path = Path(scan_dir, to_scan_file).resolve() - wfp = self.winnowing.wfp_for_file(str(abs_path), to_scan_file) - if wfp is None or wfp == '': - self.print_debug(f'No WFP returned for {to_scan_file}. Skipping.') - continue - if save_wfps_for_print: - wfp_list.append(wfp) - file_count += 1 - if self.threaded_scan: - wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, - # add it to the queue - if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - scan_block += wfp - scan_size = len(scan_block.encode('utf-8')) - wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 - if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do - scan_started = True - if not self.threaded_scan.run(wait=False): - self.print_stderr('Warning: Some errors encounted while scanning. Results might be incomplete.') - success = False - # End for loop - if self.threaded_scan and scan_block != '': - self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted - if spinner: - spinner.finish() + spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext() + + with spinner_ctx as spinner: + save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan + wfp_list = [] + scan_block = '' + scan_size = 0 + queue_size = 0 + file_count = 0 # count all files fingerprinted + wfp_file_count = 0 # count number of files in each queue post + scan_started = False + + to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) + for to_scan_file in to_scan_files: + if self.threaded_scan and self.threaded_scan.stop_scanning(): + self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') + break + self.print_debug(f'Fingerprinting {to_scan_file}...') + if spinner: + spinner.next() + abs_path = Path(scan_dir, to_scan_file).resolve() + wfp = self.winnowing.wfp_for_file(str(abs_path), to_scan_file) + if wfp is None or wfp == '': + self.print_debug(f'No WFP returned for {to_scan_file}. Skipping.') + continue + if save_wfps_for_print: + wfp_list.append(wfp) + file_count += 1 + if self.threaded_scan: + wfp_size = len(wfp.encode('utf-8')) + # If the WFP is bigger than the max post size and we already have something stored in the scan block, + # add it to the queue + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + scan_block += wfp + scan_size = len(scan_block.encode('utf-8')) + wfp_file_count += 1 + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do + scan_started = True + if not self.threaded_scan.run(wait=False): + self.print_stderr('Warning: Some errors encounted while scanning. Results might be incomplete.') + success = False + # End for loop + if self.threaded_scan and scan_block != '': + self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted if file_count > 0: if save_wfps_for_print: # Write a WFP file if no threading is requested @@ -631,63 +630,61 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 skip_extensions=self.skip_extensions, operation_type='scanning', ) - spinner = None - if not self.quiet and self.isatty: - spinner = Spinner('Fingerprinting ') - save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan - wfp_list = [] - scan_block = '' - scan_size = 0 - queue_size = 0 - file_count = 0 # count all files fingerprinted - wfp_file_count = 0 # count number of files in each queue post - scan_started = False - - to_scan_files = file_filters.get_filtered_files_from_files(files) - for file in to_scan_files: - if self.threaded_scan and self.threaded_scan.stop_scanning(): - self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') - break - self.print_debug(f'Fingerprinting {file}...') - if spinner: - spinner.next() - wfp = self.winnowing.wfp_for_file(file, file) - if wfp is None or wfp == '': - self.print_debug(f'No WFP returned for {file}. Skipping.') - continue - if save_wfps_for_print: - wfp_list.append(wfp) - file_count += 1 - if self.threaded_scan: - wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue # noqa: E501 - if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - scan_block += wfp - scan_size = len(scan_block.encode('utf-8')) - wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 - if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) - queue_size += 1 - scan_block = '' - wfp_file_count = 0 - if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do - scan_started = True - if not self.threaded_scan.run(wait=False): - self.print_stderr( - 'Warning: Some errors encounted while scanning. Results might be incomplete.' - ) - success = False + spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext() + + with spinner_ctx as spinner: + save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan + wfp_list = [] + scan_block = '' + scan_size = 0 + queue_size = 0 + file_count = 0 # count all files fingerprinted + wfp_file_count = 0 # count number of files in each queue post + scan_started = False + + to_scan_files = file_filters.get_filtered_files_from_files(files) + for file in to_scan_files: + if self.threaded_scan and self.threaded_scan.stop_scanning(): + self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.') + break + self.print_debug(f'Fingerprinting {file}...') + if spinner: + spinner.next() + wfp = self.winnowing.wfp_for_file(file, file) + if wfp is None or wfp == '': + self.print_debug(f'No WFP returned for {file}. Skipping.') + continue + if save_wfps_for_print: + wfp_list.append(wfp) + file_count += 1 + if self.threaded_scan: + wfp_size = len(wfp.encode('utf-8')) + # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue # noqa: E501 + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + scan_block += wfp + scan_size = len(scan_block.encode('utf-8')) + wfp_file_count += 1 + # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: + self.threaded_scan.queue_add(scan_block) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do + scan_started = True + if not self.threaded_scan.run(wait=False): + self.print_stderr( + 'Warning: Some errors encounted while scanning. Results might be incomplete.' + ) + success = False - # End for loop - if self.threaded_scan and scan_block != '': - self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted - if spinner: - spinner.finish() + # End for loop + if self.threaded_scan and scan_block != '': + self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted if file_count > 0: if save_wfps_for_print: # Write a WFP file if no threading is requested @@ -778,73 +775,72 @@ def scan_wfp_file(self, file: str = None) -> bool: # noqa: PLR0912, PLR0915 self.print_debug(f'Found {file_count} files to process.') raw_output = '{\n' file_print = '' - bar = None - if not self.quiet and self.isatty: - bar = Bar('Scanning', max=file_count) - bar.next(0) - with open(wfp_file) as f: - for line in f: - if line.startswith(WFP_FILE_START): - if file_print: - wfp += file_print # Store the WFP for the current file - cur_size = len(wfp.encode('utf-8')) - file_print = line # Start storing the next file - cur_files += 1 - batch_files += 1 - else: - file_print += line # Store the rest of the WFP for this file - l_size = cur_size + len(file_print.encode('utf-8')) - # Hit the max post size, so sending the current batch and continue processing - if l_size >= self.max_post_size and wfp: - self.print_debug( - f'Sending {batch_files} ({cur_files}) of' - f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' - ) - if self.debug and cur_size > self.max_post_size: - Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') - scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store - if bar: - bar.next(batch_files) - if scan_resp is not None: - for key, value in scan_resp.items(): - raw_output += ' "%s":%s,' % (key, json.dumps(value, indent=2)) - for v in value: - if hasattr(v, 'get'): - if v.get('id') != 'none': - vcv = '%s:%s:%s' % (v.get('vendor'), v.get('component'), v.get('version')) - components[vcv] = components[vcv] + 1 if vcv in components else 1 - if max_component['hits'] < components[vcv]: - max_component['name'] = v.get('component') - max_component['hits'] = components[vcv] - else: - Scanner.print_stderr(f'Warning: Unknown value: {v}') - else: - success = False - batch_files = 0 - wfp = '' - if file_print: - wfp += file_print # Store the WFP for the current file - if wfp: - self.print_debug( - f'Sending {batch_files} ({cur_files}) of' - f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' - ) - scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store + bar_ctx = Bar('Scanning', max=file_count) if (not self.quiet and self.isatty) else nullcontext() + + with bar_ctx as bar: if bar: - bar.next(batch_files) - first = True - if scan_resp is not None: - for key, value in scan_resp.items(): - if first: - raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2)) - first = False + bar.next(0) + with open(wfp_file) as f: + for line in f: + if line.startswith(WFP_FILE_START): + if file_print: + wfp += file_print # Store the WFP for the current file + cur_size = len(wfp.encode('utf-8')) + file_print = line # Start storing the next file + cur_files += 1 + batch_files += 1 else: - raw_output += ',\n "%s":%s' % (key, json.dumps(value, indent=2)) - else: - success = False + file_print += line # Store the rest of the WFP for this file + l_size = cur_size + len(file_print.encode('utf-8')) + # Hit the max post size, so sending the current batch and continue processing + if l_size >= self.max_post_size and wfp: + self.print_debug( + f'Sending {batch_files} ({cur_files}) of' + f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' + ) + if self.debug and cur_size > self.max_post_size: + Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') + scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store + if bar: + bar.next(batch_files) + if scan_resp is not None: + for key, value in scan_resp.items(): + raw_output += ' "%s":%s,' % (key, json.dumps(value, indent=2)) + for v in value: + if hasattr(v, 'get'): + if v.get('id') != 'none': + vcv = '%s:%s:%s' % (v.get('vendor'), v.get('component'), v.get('version')) + components[vcv] = components[vcv] + 1 if vcv in components else 1 + if max_component['hits'] < components[vcv]: + max_component['name'] = v.get('component') + max_component['hits'] = components[vcv] + else: + Scanner.print_stderr(f'Warning: Unknown value: {v}') + else: + success = False + batch_files = 0 + wfp = '' + if file_print: + wfp += file_print # Store the WFP for the current file + if wfp: + self.print_debug( + f'Sending {batch_files} ({cur_files}) of' + f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' + ) + scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store + if bar: + bar.next(batch_files) + first = True + if scan_resp is not None: + for key, value in scan_resp.items(): + if first: + raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2)) + first = False + else: + raw_output += ',\n "%s":%s' % (key, json.dumps(value, indent=2)) + else: + success = False raw_output += '\n}' - if bar: - bar.finish() if self.output_format == 'plain': self.__log_result(raw_output) elif self.output_format == 'cyclonedx': @@ -1052,19 +1048,16 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): ) wfps = '' self.print_msg(f'Searching {scan_dir} for files to fingerprint...') - spinner = None - if not self.quiet and self.isatty: - spinner = Spinner('Fingerprinting ') - - to_fingerprint_files = file_filters.get_filtered_files_from_folder(scan_dir) - for file in to_fingerprint_files: - if spinner: - spinner.next() - abs_path = Path(scan_dir, file).resolve() - self.print_debug(f'Fingerprinting {file}...') - wfps += self.winnowing.wfp_for_file(str(abs_path), file) - if spinner: - spinner.finish() + spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext() + + with spinner_ctx as spinner: + to_fingerprint_files = file_filters.get_filtered_files_from_folder(scan_dir) + for file in to_fingerprint_files: + if spinner: + spinner.next() + abs_path = Path(scan_dir, file).resolve() + self.print_debug(f'Fingerprinting {file}...') + wfps += self.winnowing.wfp_for_file(str(abs_path), file) if wfps: if wfp_file: self.print_stderr(f'Writing fingerprints to {wfp_file}') diff --git a/src/scanoss/scanners/folder_hasher.py b/src/scanoss/scanners/folder_hasher.py index eb4bd726..549a7c18 100644 --- a/src/scanoss/scanners/folder_hasher.py +++ b/src/scanoss/scanners/folder_hasher.py @@ -157,38 +157,38 @@ def _build_root_node( # Sort the files by name to ensure the hash is the same for the same folder filtered_files.sort() - bar = Bar('Hashing files...', max=len(filtered_files)) - full_file_path = '' - for file_path in filtered_files: - try: - file_path_obj = Path(file_path) if isinstance(file_path, str) else file_path - full_file_path = file_path_obj if file_path_obj.is_absolute() else root / file_path_obj + bar_ctx = Bar('Hashing files...', max=len(filtered_files)) - self.base.print_debug(f'\nHashing file {str(full_file_path)}') + with bar_ctx as bar: + full_file_path = '' + for file_path in filtered_files: + try: + file_path_obj = Path(file_path) if isinstance(file_path, str) else file_path + full_file_path = file_path_obj if file_path_obj.is_absolute() else root / file_path_obj - file_bytes = full_file_path.read_bytes() - key = CRC64.get_hash_buff(file_bytes) - key_str = ''.join(f'{b:02x}' for b in key) - rel_path = str(full_file_path.relative_to(root)) + self.base.print_debug(f'\nHashing file {str(full_file_path)}') - file_item = DirectoryFile(rel_path, key, key_str) + file_bytes = full_file_path.read_bytes() + key = CRC64.get_hash_buff(file_bytes) + key_str = ''.join(f'{b:02x}' for b in key) + rel_path = str(full_file_path.relative_to(root)) - current_node = root_node - for part in Path(rel_path).parent.parts: - child_path = str(Path(current_node.path) / part) - if child_path not in current_node.children: - current_node.children[child_path] = DirectoryNode(child_path) - current_node = current_node.children[child_path] - current_node.files.append(file_item) + file_item = DirectoryFile(rel_path, key, key_str) - root_node.files.append(file_item) + current_node = root_node + for part in Path(rel_path).parent.parts: + child_path = str(Path(current_node.path) / part) + if child_path not in current_node.children: + current_node.children[child_path] = DirectoryNode(child_path) + current_node = current_node.children[child_path] + current_node.files.append(file_item) - except Exception as e: - self.base.print_debug(f'Skipping file {full_file_path}: {str(e)}') + root_node.files.append(file_item) - bar.next() + except Exception as e: + self.base.print_debug(f'Skipping file {full_file_path}: {str(e)}') - bar.finish() + bar.next() return root_node def _hash_calc_from_node(self, node: DirectoryNode, current_depth: int = 1) -> dict: diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 7ac64630..8d4a2849 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -110,6 +110,19 @@ def __init__( # noqa: PLR0913 self.min_accepted_score = min_accepted_score self.use_grpc = use_grpc + def _execute_grpc_scan(self, hfh_request: Dict) -> None: + """ + Execute folder hash scan. + + Args: + hfh_request: Request dictionary for the gRPC call + """ + try: + self.scan_results = self.client.folder_hash_scan(hfh_request, self.use_grpc) + except Exception as e: + self.base.print_stderr(f'Error during folder hash scan: {e}') + self.scan_results = None + def scan(self) -> Optional[Dict]: """ Scan the provided directory using the folder hashing algorithm. @@ -124,25 +137,17 @@ def scan(self) -> Optional[Dict]: 'min_accepted_score': self.min_accepted_score, } - spinner = Spinner('Scanning folder...') - stop_spinner = False + spinner_ctx = Spinner('Scanning folder...') + + with spinner_ctx as spinner: + grpc_thread = threading.Thread(target=self._execute_grpc_scan, args=(hfh_request,)) + grpc_thread.start() - def spin(): - while not stop_spinner: + while grpc_thread.is_alive(): spinner.next() time.sleep(0.1) - spinner_thread = threading.Thread(target=spin) - spinner_thread.start() - - try: - response = self.client.folder_hash_scan(hfh_request, self.use_grpc) - if response: - self.scan_results = response - finally: - stop_spinner = True - spinner_thread.join() - spinner.finish() + grpc_thread.join() return self.scan_results diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index e9784e3a..d0a5cad7 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -22,6 +22,7 @@ THE SOFTWARE. """ +import atexit import os import queue import sys @@ -77,6 +78,8 @@ def __init__( if nb_threads > MAX_ALLOWED_THREADS: self.print_msg(f'Warning: Requested threads too large: {nb_threads}. Reducing to {MAX_ALLOWED_THREADS}') self.nb_threads = MAX_ALLOWED_THREADS + # Register cleanup to ensure progress bar is finished on exit + atexit.register(self.complete_bar) @staticmethod def __count_files_in_wfp(wfp: str): @@ -101,6 +104,13 @@ def complete_bar(self): if self.bar: self.bar.finish() + def __del__(self): + """Ensure progress bar is cleaned up when object is destroyed""" + try: + self.complete_bar() + except Exception: + pass # Ignore errors during cleanup + def set_bar(self, bar: Bar) -> None: """ Set the Progress Bar to display progress while scanning From 73075fdfa27e8749cc51a0c667c940e9cd188b0a Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 13 Nov 2025 16:31:05 +0100 Subject: [PATCH 403/489] feat(policies): add support for license source filtering in copyleft inspection --- CHANGELOG.md | 11 + src/scanoss/cli.py | 14 + src/scanoss/constants.py | 3 + .../policy_check/scanoss/copyleft.py | 8 +- .../inspection/utils/scan_result_processor.py | 33 ++- tests/test_policy_inspect.py | 254 ++++++++++++++++++ 6 files changed, 311 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cc984f0..de72021d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added +- Added `--license-sources` (`-ls`) option to copyleft inspection + - Filter which license sources to check (component_declared, license_file, file_header, file_spdx_tag, scancode) + - Supports both `-ls source1 source2` and `-ls source1 -ls source2` syntax + +### Changed +- Copyleft inspection now defaults to component-level licenses only (component_declared, license_file) + - Reduces noise from file-level license detections (file_header, scancode) + - Use `-ls` to override and check specific sources + ### Fixed - Fixed terminal cursor disappearing after aborting scan with Ctrl+C diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 824c5133..0a3c1ae0 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -55,6 +55,7 @@ from .components import Components from .constants import ( DEFAULT_API_TIMEOUT, + DEFAULT_COPYLEFT_LICENSE_SOURCES, DEFAULT_HFH_DEPTH, DEFAULT_HFH_MIN_ACCEPTED_SCORE, DEFAULT_HFH_RANK_THRESHOLD, @@ -64,6 +65,7 @@ DEFAULT_TIMEOUT, MIN_TIMEOUT, PYTHON_MAJOR_VERSION, + VALID_LICENSE_SOURCES, ) from .csvoutput import CsvOutput from .cyclonedx import CycloneDx @@ -699,6 +701,17 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p.add_argument('--exclude', help='Licenses to exclude from analysis (comma-separated list)') p.add_argument('--explicit', help='Use only these specific licenses for analysis (comma-separated list)') + # License source filtering + for p in [p_inspect_raw_copyleft, p_inspect_legacy_copyleft]: + p.add_argument( + '-ls', '--license-sources', + action='extend', + nargs='+', + choices=VALID_LICENSE_SOURCES, + help=f'Specify which license sources to check for copyleft violations. Each license object in scan results ' + f'has a source field indicating its origin. Default: {", ".join(DEFAULT_COPYLEFT_LICENSE_SOURCES)}', + ) + # Common options for (legacy) copyleft and undeclared component inspection for p in [p_inspect_raw_copyleft, p_inspect_raw_undeclared, p_inspect_legacy_copyleft, p_inspect_legacy_undeclared]: p.add_argument('-i', '--input', nargs='?', help='Path to scan results file to analyse') @@ -1752,6 +1765,7 @@ def inspect_copyleft(parser, args): include=args.include, # Additional licenses to check exclude=args.exclude, # Licenses to ignore explicit=args.explicit, # Explicit license list + license_sources=args.license_sources, # License sources to check (list) ) # Execute inspection and exit with appropriate status code status, _ = i_copyleft.run() diff --git a/src/scanoss/constants.py b/src/scanoss/constants.py index 989f2008..68216595 100644 --- a/src/scanoss/constants.py +++ b/src/scanoss/constants.py @@ -17,3 +17,6 @@ DEFAULT_HFH_DEPTH = 1 DEFAULT_HFH_RECURSIVE_THRESHOLD = 0.8 DEFAULT_HFH_MIN_ACCEPTED_SCORE = 0.15 + +VALID_LICENSE_SOURCES = ['component_declared', 'license_file', 'file_header', 'file_spdx_tag', 'scancode'] +DEFAULT_COPYLEFT_LICENSE_SOURCES = ['component_declared', 'license_file'] diff --git a/src/scanoss/inspection/policy_check/scanoss/copyleft.py b/src/scanoss/inspection/policy_check/scanoss/copyleft.py index 08694854..a56c39b9 100644 --- a/src/scanoss/inspection/policy_check/scanoss/copyleft.py +++ b/src/scanoss/inspection/policy_check/scanoss/copyleft.py @@ -26,6 +26,8 @@ from dataclasses import dataclass from typing import Dict, List +from scanoss.constants import DEFAULT_COPYLEFT_LICENSE_SOURCES + from ...policy_check.policy_check import PolicyCheck, PolicyOutput, PolicyStatus from ...utils.markdown_utils import generate_jira_table, generate_table from ...utils.scan_result_processor import ScanResultProcessor @@ -63,6 +65,7 @@ def __init__( # noqa: PLR0913 include: str = None, exclude: str = None, explicit: str = None, + license_sources: list = None, ): """ Initialise the Copyleft class. @@ -77,6 +80,7 @@ def __init__( # noqa: PLR0913 :param include: Licenses to include in the analysis :param exclude: Licenses to exclude from the analysis :param explicit: Explicitly defined licenses + :param license_sources: List of license sources to check """ super().__init__( debug, trace, quiet, format_type, status, name='Copyleft Policy', output=output @@ -85,6 +89,7 @@ def __init__( # noqa: PLR0913 self.filepath = filepath self.output = output self.status = status + self.license_sources = license_sources or DEFAULT_COPYLEFT_LICENSE_SOURCES self.results_processor = ScanResultProcessor( self.debug, self.trace, @@ -92,7 +97,8 @@ def __init__( # noqa: PLR0913 self.filepath, include, exclude, - explicit) + explicit, + self.license_sources) def _json(self, components: list[Component]) -> PolicyOutput: """ diff --git a/src/scanoss/inspection/utils/scan_result_processor.py b/src/scanoss/inspection/utils/scan_result_processor.py index 22333b5d..75960eab 100644 --- a/src/scanoss/inspection/utils/scan_result_processor.py +++ b/src/scanoss/inspection/utils/scan_result_processor.py @@ -71,11 +71,13 @@ def __init__( # noqa: PLR0913 include: str = None, exclude: str = None, explicit: str = None, + license_sources: list = None, ): super().__init__(debug, trace, quiet) self.result_file_path = result_file_path self.license_util = LicenseUtil() self.license_util.init(include, exclude, explicit) + self.license_sources = license_sources self.results = self._load_input_file() def get_results(self) -> Dict[str, Any]: @@ -162,9 +164,11 @@ def _append_license_to_component(self, self.print_debug(f'WARNING: Results missing licenses. Skipping: {new_component}') return - licenses_order_by_source_priority = self._get_licenses_order_by_source_priority(new_component['licenses']) + # Select licenses based on configuration (filtering or priority mode) + selected_licenses = self._select_licenses(new_component['licenses']) + # Process licenses for this component - for license_item in licenses_order_by_source_priority: + for license_item in selected_licenses: if license_item.get('name'): spdxid = license_item['name'] source = license_item.get('source') @@ -309,19 +313,26 @@ def convert_components_to_list(self, components: dict): component['licenses'] = [] return results_list - def _get_licenses_order_by_source_priority(self,licenses_data): + def _select_licenses(self, licenses_data): """ - Select licenses based on source priority: - 1. component_declared (highest priority) - 2. license_file - 3. file_header - 4. scancode (lowest priority) + Select licenses based on configuration. + + Two modes: + - Filtering mode: If license_sources specified, filter to those sources + - Priority mode: Otherwise, use original priority-based selection - If any high-priority source is found, return only licenses from that source. - If none found, return all licenses. + Args: + licenses_data: List of license dictionaries - Returns: list with ordered licenses by source. + Returns: + Filtered list of licenses based on configuration """ + # Filtering mode, when license_sources is explicitly provided + if self.license_sources: + sources_to_include = set(self.license_sources) | {'unknown'} + return [lic for lic in licenses_data + if lic.get('source') in sources_to_include or lic.get('source') is None] + # Define priority order (highest to lowest) priority_sources = ['component_declared', 'license_file', 'file_header', 'scancode'] diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index 0ccf7886..a3161a45 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -28,6 +28,7 @@ import unittest from unittest.mock import Mock, patch +from scanoss.constants import DEFAULT_COPYLEFT_LICENSE_SOURCES, VALID_LICENSE_SOURCES from src.scanoss.inspection.policy_check.dependency_track.project_violation import ( DependencyTrackProjectViolationPolicyCheck, ) @@ -389,6 +390,259 @@ def test_copyleft_policy_jira_markdown_output(self): self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) self.assertEqual(expected_details_output, results) + ## Copyleft License Source Filtering Tests ## + + def test_copyleft_policy_default_license_sources(self): + """ + Test default behavior: should use DEFAULT_COPYLEFT_LICENSE_SOURCES + (component_declared and license_file) + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json') + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find components with copyleft from component_declared or license_file + # Expected: 5 PURL@version entries (scanner.c x2, engine x2, wfp x1) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertEqual(len(details['components']), 5) + + # Verify all components have licenses from default sources + for component in details['components']: + for license in component['licenses']: + self.assertIn(license['source'], DEFAULT_COPYLEFT_LICENSE_SOURCES) + + def test_copyleft_policy_license_sources_none(self): + """ + Test explicit None: should use DEFAULT_COPYLEFT_LICENSE_SOURCES + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft(filepath=input_file_name, format_type='json', license_sources=None) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should behave same as default + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertEqual(len(details['components']), 5) + + # Verify all components have licenses from default sources + for component in details['components']: + for license in component['licenses']: + self.assertIn(license['source'], DEFAULT_COPYLEFT_LICENSE_SOURCES) + + + def test_copyleft_policy_license_sources_component_declared_only(self): + """ + Test filtering to component_declared source only + Should find GPL-2.0-only from component_declared + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=['component_declared'] + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find 5 PURL@version entries from component_declared + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertEqual(len(details['components']), 5) + + # All licenses should be from component_declared + for component in details['components']: + for license in component['licenses']: + self.assertEqual(license['source'], 'component_declared') + + def test_copyleft_policy_license_sources_license_file_only(self): + """ + Test filtering to license_file source only + Should find GPL-2.0-only from license_file (engine and wfp) + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=['license_file'] + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find engine and wfp (2 components with license_file) + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertEqual(len(details['components']), 2) + + # Verify components are engine and wfp + purls = [comp['purl'] for comp in details['components']] + self.assertIn('pkg:github/scanoss/engine', purls) + self.assertIn('pkg:github/scanoss/wfp', purls) + + # All licenses should be from license_file + for component in details['components']: + for license in component['licenses']: + self.assertEqual(license['source'], 'license_file') + + def test_copyleft_policy_license_sources_file_header_only(self): + """ + Test filtering to file_header source only + file_header only has BSD-2-Clause and Zlib (not copyleft) + Should find no copyleft licenses + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=['file_header'] + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find no copyleft (file_header only has BSD and Zlib) + self.assertEqual(status, PolicyStatus.POLICY_SUCCESS.value) + self.assertEqual(details, {}) + + def test_copyleft_policy_license_sources_multiple_sources(self): + """ + Test using multiple license sources + Should find copyleft from component_declared and scancode + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=['component_declared', 'scancode'] + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find components from both sources + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertGreaterEqual(len(details['components']), 3) + + # Verify licenses are from specified sources + for component in details['components']: + for license in component['licenses']: + self.assertIn(license['source'], ['component_declared', 'scancode']) + + def test_copyleft_policy_license_sources_all_valid_sources(self): + """ + Test using all valid license sources + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=VALID_LICENSE_SOURCES + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find all copyleft licenses from any source + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertGreaterEqual(len(details['components']), 3) + + def test_copyleft_policy_license_sources_with_markdown_output(self): + """ + Test license source filtering works with markdown output + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='md', + license_sources=['license_file'] + ) + status, policy_output = copyleft.run() + + # Should generate markdown table + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + self.assertIn('### Copyleft Licenses', policy_output.details) + self.assertIn('Component', policy_output.details) + self.assertIn('License', policy_output.details) + self.assertIn('2 component(s) with copyleft licenses were found', policy_output.summary) + + def test_copyleft_policy_license_sources_with_include_filter(self): + """ + Test license_sources works with include filter + Filter to scancode source and include only GPL-2.0-or-later + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=['scancode'], + include='GPL-2.0-or-later' + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find only GPL-2.0-or-later from scancode + self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) + if details: # May be empty if no matches + for component in details.get('components', []): + for license in component['licenses']: + self.assertEqual(license['spdxid'], 'GPL-2.0-or-later') + self.assertEqual(license['source'], 'scancode') + + def test_copyleft_policy_license_sources_with_exclude_filter(self): + """ + Test license_sources works with exclude filter + Use component_declared but exclude GPL-2.0-only + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=['component_declared'], + exclude='GPL-2.0-only' + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should exclude GPL-2.0-only, leaving nothing (all component_declared are GPL-2.0-only) + self.assertEqual(status, PolicyStatus.POLICY_SUCCESS.value) + self.assertEqual(details, {}) + + def test_copyleft_policy_license_sources_no_copyleft_file(self): + """ + Test license_sources with result-no-copyleft.json + Should return success even with license_sources specified + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + file_name = 'result-no-copyleft.json' + input_file_name = os.path.join(script_dir, 'data', file_name) + copyleft = Copyleft( + filepath=input_file_name, + format_type='json', + license_sources=['component_declared'] + ) + status, policy_output = copyleft.run() + details = json.loads(policy_output.details) + + # Should find no copyleft + self.assertEqual(status, PolicyStatus.POLICY_SUCCESS.value) + self.assertEqual(details, {}) + self.assertIn('0 component(s) with copyleft licenses were found', policy_output.summary) + def test_inspect_license_summary(self): script_dir = os.path.dirname(os.path.abspath(__file__)) file_name = 'result.json' From c7e8e35916431ef559a6048457826e6599ddf98c Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 17 Nov 2025 11:17:42 +0100 Subject: [PATCH 404/489] feat(licenses): integrate OSADL copyleft data --- CHANGELOG.md | 9 +- README.md | 5 + src/scanoss/data/osadl-copyleft.json | 133 ++++++++++++++++++ src/scanoss/inspection/utils/license_utils.py | 128 ++++++++--------- src/scanoss/osadl.py | 125 ++++++++++++++++ tests/test_osadl.py | 102 ++++++++++++++ tests/test_policy_inspect.py | 17 +-- 7 files changed, 439 insertions(+), 80 deletions(-) create mode 100644 src/scanoss/data/osadl-copyleft.json create mode 100644 src/scanoss/osadl.py create mode 100644 tests/test_osadl.py diff --git a/CHANGELOG.md b/CHANGELOG.md index de72021d..532ba071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Supports both `-ls source1 source2` and `-ls source1 -ls source2` syntax ### Changed +- **Switched to OSADL authoritative copyleft license data** + - Copyleft detection now uses [OSADL (Open Source Automation Development Lab)](https://www.osadl.org/) checklist data + - Adds missing `-or-later` license variants (GPL-2.0-or-later, GPL-3.0-or-later, LGPL-2.1-or-later, etc.) + - Expands copyleft coverage from 21 to 32 licenses + - Custom include/exclude/explicit filters still use legacy behavior for backward compatibility + - Dataset attribution added to README (CC-BY-4.0 license) + - Copyleft inspection now defaults to component-level licenses only (component_declared, license_file) - Reduces noise from file-level license detections (file_header, scancode) - Use `-ls` to override and check specific sources ### Fixed -- Fixed terminal cursor disappearing after aborting scan with Ctrl+C +- Fixed the terminal cursor disappearing after aborting scan with Ctrl+C ## [1.40.1] - 2025-10-29 ### Changed diff --git a/README.md b/README.md index f5d1bfb2..ececd218 100644 --- a/README.md +++ b/README.md @@ -135,3 +135,8 @@ Details of major changes to the library can be found in [CHANGELOG.md](CHANGELOG ## Background Details about the Winnowing algorithm used for scanning can be found [here](WINNOWING.md). + +## Dataset License Notice +This application is licensed under the MIT License. In addition, it includes an unmodified copy of the OSADL copyleft license dataset ([osadl-copyleft.json](src/scanoss/data/osadl-copyleft.json)) which is licensed under the [Creative Commons Attribution 4.0 International license (CC-BY-4.0)](https://creativecommons.org/licenses/by/4.0/) by the [Open Source Automation Development Lab (OSADL) eG](https://www.osadl.org/). + +**Attribution:** A project by the Open Source Automation Development Lab (OSADL) eG. Original source: [https://www.osadl.org/fileadmin/checklists/copyleft.json](https://www.osadl.org/fileadmin/checklists/copyleft.json) diff --git a/src/scanoss/data/osadl-copyleft.json b/src/scanoss/data/osadl-copyleft.json new file mode 100644 index 00000000..6cc2e1a7 --- /dev/null +++ b/src/scanoss/data/osadl-copyleft.json @@ -0,0 +1,133 @@ +{ + "title": "OSADL Open Source License Obligations Checklist (https:\/\/www.osadl.org\/Checklists)", + "license": "Creative Commons Attribution 4.0 International license (CC-BY-4.0)", + "attribution": "A project by the Open Source Automation Development Lab (OSADL) eG. For further information about the project see the description at www.osadl.org\/checklists.", + "copyright": "(C) 2017 - 2024 Open Source Automation Development Lab (OSADL) eG and contributors, info@osadl.org", + "disclaimer": "The checklists and particularly the copyleft data have been assembled with maximum diligence and care; however, the authors do not warrant nor can be held liable in any way for its correctness, usefulness, merchantibility or fitness for a particular purpose as far as permissible by applicable law. Anyone who uses the information does this on his or her sole responsibility. For any individual legal advice, it is recommended to contact a lawyer.", + "timeformat": "%Y-%m-%dT%H:%M:%S%z", + "timestamp": "2025-10-30T11:23:00+0000", + "copyleft": + { + "0BSD": "No", + "AFL-2.0": "No", + "AFL-2.1": "No", + "AFL-3.0": "No", + "AGPL-3.0-only": "Yes", + "AGPL-3.0-or-later": "Yes", + "Apache-1.0": "No", + "Apache-1.1": "No", + "Apache-2.0": "No", + "APSL-2.0": "Yes (restricted)", + "Artistic-1.0": "No", + "Artistic-1.0-Perl": "No", + "Artistic-2.0": "No", + "Bitstream-Vera": "No", + "blessing": "No", + "BlueOak-1.0.0": "No", + "BSD-1-Clause": "No", + "BSD-2-Clause": "No", + "BSD-2-Clause-Patent": "No", + "BSD-3-Clause": "No", + "BSD-3-Clause-Open-MPI": "No", + "BSD-4-Clause": "No", + "BSD-4-Clause-UC": "No", + "BSD-4.3TAHOE": "No", + "BSD-Source-Code": "No", + "BSL-1.0": "No", + "bzip2-1.0.5": "No", + "bzip2-1.0.6": "No", + "CC-BY-2.5": "No", + "CC-BY-3.0": "No", + "CDDL-1.0": "Yes (restricted)", + "CDDL-1.1": "Yes (restricted)", + "CPL-1.0": "Yes", + "curl": "No", + "ECL-1.0": "No", + "ECL-2.0": "No", + "EFL-2.0": "No", + "EPL-1.0": "Yes", + "EPL-2.0": "Yes (restricted)", + "EUPL-1.1": "Yes", + "EUPL-1.2": "Yes", + "FSFAP": "No", + "FSFUL": "No", + "FSFULLR": "No", + "FSFULLRWD": "No", + "FTL": "No", + "GPL-1.0-only": "Yes", + "GPL-1.0-or-later": "Yes", + "GPL-2.0-only": "Yes", + "GPL-2.0-only WITH Classpath-exception-2.0": "Yes (restricted)", + "GPL-2.0-or-later": "Yes", + "GPL-3.0-only": "Yes", + "GPL-3.0-or-later": "Yes", + "HPND": "No", + "IBM-pibs": "No", + "ICU": "No", + "IJG": "No", + "ImageMagick": "No", + "Info-ZIP": "No", + "IPL-1.0": "Yes", + "ISC": "No", + "JasPer-2.0": "No", + "LGPL-2.0-only": "Yes (restricted)", + "LGPL-2.0-or-later": "Yes (restricted)", + "LGPL-2.1-only": "Yes (restricted)", + "LGPL-2.1-or-later": "Yes (restricted)", + "LGPL-3.0-only": "Yes (restricted)", + "LGPL-3.0-or-later": "Yes (restricted)", + "Libpng": "No", + "libpng-2.0": "No", + "libtiff": "No", + "LicenseRef-scancode-bsla-no-advert": "No", + "LicenseRef-scancode-info-zip-2003-05": "No", + "LicenseRef-scancode-ppp": "No", + "Minpack": "No", + "MirOS": "No", + "MIT": "No", + "MIT-0": "No", + "MIT-CMU": "No", + "MPL-1.1": "Yes (restricted)", + "MPL-2.0": "Yes (restricted)", + "MPL-2.0-no-copyleft-exception": "Yes (restricted)", + "MS-PL": "Questionable", + "MS-RL": "Yes (restricted)", + "NBPL-1.0": "No", + "NCSA": "No", + "NTP": "No", + "OFL-1.1": "Yes (restricted)", + "OGC-1.0": "No", + "OLDAP-2.8": "No", + "OpenSSL": "Questionable", + "OSL-3.0": "Yes", + "PHP-3.01": "No", + "PostgreSQL": "No", + "PSF-2.0": "No", + "Python-2.0": "No", + "Qhull": "No", + "RSA-MD": "No", + "Saxpath": "No", + "SGI-B-2.0": "No", + "Sleepycat": "Yes", + "SMLNJ": "No", + "Spencer-86": "No", + "SSH-OpenSSH": "No", + "SSH-short": "No", + "SunPro": "No", + "Ubuntu-font-1.0": "Yes (restricted)", + "Unicode-3.0": "No", + "Unicode-DFS-2015": "No", + "Unicode-DFS-2016": "No", + "Unlicense": "No", + "UPL-1.0": "No", + "W3C": "No", + "W3C-19980720": "No", + "W3C-20150513": "No", + "WTFPL": "No", + "X11": "No", + "XFree86-1.1": "No", + "Zlib": "No", + "zlib-acknowledgement": "No", + "ZPL-2.0": "No" + } +} \ No newline at end of file diff --git a/src/scanoss/inspection/utils/license_utils.py b/src/scanoss/inspection/utils/license_utils.py index beb7dd09..fd4fec38 100644 --- a/src/scanoss/inspection/utils/license_utils.py +++ b/src/scanoss/inspection/utils/license_utils.py @@ -22,96 +22,90 @@ THE SOFTWARE. """ -from ...scanossbase import ScanossBase +from scanoss.osadl import Osadl -DEFAULT_COPYLEFT_LICENSES = { - 'agpl-3.0-only', - 'artistic-1.0', - 'artistic-2.0', - 'cc-by-sa-4.0', - 'cddl-1.0', - 'cddl-1.1', - 'cecill-2.1', - 'epl-1.0', - 'epl-2.0', - 'gfdl-1.1-only', - 'gfdl-1.2-only', - 'gfdl-1.3-only', - 'gpl-1.0-only', - 'gpl-2.0-only', - 'gpl-3.0-only', - 'lgpl-2.1-only', - 'lgpl-3.0-only', - 'mpl-1.1', - 'mpl-2.0', - 'sleepycat', - 'watcom-1.0', -} +from ...scanossbase import ScanossBase class LicenseUtil(ScanossBase): """ A utility class for handling software licenses, particularly copyleft licenses. - This class provides functionality to initialize, manage, and query a set of - copyleft licenses. It also offers a method to generate URLs for license information. + Uses OSADL (Open Source Automation Development Lab) authoritative copyleft data + with optional include/exclude/explicit filters. """ BASE_SPDX_ORG_URL = 'https://spdx.org/licenses' - BASE_OSADL_URL = 'https://www.osadl.org/fileadmin/checklists/unreflicenses' def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False): super().__init__(debug, trace, quiet) - self.default_copyleft_licenses = set(DEFAULT_COPYLEFT_LICENSES) - self.copyleft_licenses = set() + self.osadl = Osadl(debug=debug, trace=trace, quiet=quiet) + self.include_licenses = set() + self.exclude_licenses = set() + self.explicit_licenses = set() def init(self, include: str = None, exclude: str = None, explicit: str = None): """ - Initialize the set of copyleft licenses based on user input. - - This method allows for customization of the copyleft license set by: - - Setting an explicit list of licenses - - Including additional licenses to the default set - - Excluding specific licenses from the default set + Initialize copyleft license filters. - :param include: Comma-separated string of licenses to include - :param exclude: Comma-separated string of licenses to exclude - :param explicit: Comma-separated string of licenses to use exclusively + :param include: Comma-separated licenses to mark as copyleft (in addition to OSADL) + :param exclude: Comma-separated licenses to mark as NOT copyleft (override OSADL) + :param explicit: Comma-separated licenses to use exclusively (ignore OSADL) """ - if self.debug: - self.print_stderr(f'Include Copyleft licenses: ${include}') - self.print_stderr(f'Exclude Copyleft licenses: ${exclude}') - self.print_stderr(f'Explicit Copyleft licenses: ${explicit}') - if explicit: - explicit = explicit.strip() + # Reset previous filters so init() can be safely called multiple times + self.include_licenses.clear() + self.exclude_licenses.clear() + self.explicit_licenses.clear() + + # Parse explicit list (if provided, ignore OSADL completely) if explicit: - exp = [item.strip().lower() for item in explicit.split(',')] - self.copyleft_licenses = set(exp) - self.print_debug(f'Copyleft licenses: ${self.copyleft_licenses}') + self.explicit_licenses = {lic.strip().lower() for lic in explicit.split(',') if lic.strip()} + self.print_debug(f'Explicit copyleft licenses: {self.explicit_licenses}') return - # If no explicit licenses were set, set default ones - self.copyleft_licenses = self.default_copyleft_licenses.copy() - if include: - include = include.strip() + + # Parse include list (mark these as copyleft in addition to OSADL) if include: - inc = [item.strip().lower() for item in include.split(',')] - self.copyleft_licenses.update(inc) - if exclude: - exclude = exclude.strip() + self.include_licenses = {lic.strip().lower() for lic in include.split(',') if lic.strip()} + self.print_debug(f'Include licenses: {self.include_licenses}') + + # Parse exclude list (mark these as NOT copyleft, overriding OSADL) if exclude: - inc = [item.strip().lower() for item in exclude.split(',')] - for lic in inc: - self.copyleft_licenses.discard(lic) - self.print_debug(f'Copyleft licenses: ${self.copyleft_licenses}') + self.exclude_licenses = {lic.strip().lower() for lic in exclude.split(',') if lic.strip()} + self.print_debug(f'Exclude licenses: {self.exclude_licenses}') def is_copyleft(self, spdxid: str) -> bool: """ - Check if a given license is considered copyleft. + Check if a license is copyleft. + + Logic: + 1. If explicit list provided → check if license in explicit list + 2. If license in include list → return True + 3. If license in exclude list → return False + 4. Otherwise → use OSADL authoritative data - :param spdxid: The SPDX identifier of the license to check - :return: True if the license is copyleft, False otherwise + :param spdxid: SPDX license identifier + :return: True if copyleft, False otherwise """ - return spdxid.lower() in self.copyleft_licenses + if not spdxid: + self.print_debug('No license ID provided for copyleft check') + return False + + spdxid_lc = spdxid.lower() + + # Explicit mode: use only the explicit list + if self.explicit_licenses: + return spdxid_lc in self.explicit_licenses + + # Include filter: if license in include list, force copyleft=True + if spdxid_lc in self.include_licenses: + return True + + # Exclude filter: if license in exclude list, force copyleft=False + if spdxid_lc in self.exclude_licenses: + return False + + # No filters matched, use OSADL authoritative data + return self.osadl.is_copyleft(spdxid) def get_spdx_url(self, spdxid: str) -> str: """ @@ -122,14 +116,6 @@ def get_spdx_url(self, spdxid: str) -> str: """ return f'{self.BASE_SPDX_ORG_URL}/{spdxid}.html' - def get_osadl_url(self, spdxid: str) -> str: - """ - Generate the URL for the OSADL (Open Source Automation Development Lab) page of a license. - - :param spdxid: The SPDX identifier of the license - :return: The URL of the OSADL page for the given license - """ - return f'{self.BASE_OSADL_URL}/{spdxid}.txt' # diff --git a/src/scanoss/osadl.py b/src/scanoss/osadl.py new file mode 100644 index 00000000..36f68b90 --- /dev/null +++ b/src/scanoss/osadl.py @@ -0,0 +1,125 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import sys + +import importlib_resources + +from scanoss.scanossbase import ScanossBase + + +class Osadl(ScanossBase): + """ + OSADL data accessor class. + + Provides access to OSADL (Open Source Automation Development Lab) authoritative + checklist data for license analysis. + + Data is loaded once at class level and shared across all instances for efficiency. + + Data source: https://www.osadl.org/fileadmin/checklists/copyleft.json + License: CC-BY-4.0 + """ + + _shared_copyleft_data = {} + _data_loaded = False + + def __init__(self, debug: bool = False, trace: bool = True, quiet: bool = False): + """ + Initialize the Osadl class. + Data is loaded once at class level and shared across all instances. + """ + super().__init__(debug, trace, quiet) + self._load_copyleft_data() + + + def _load_copyleft_data(self) -> bool: + """ + Load the embedded OSADL copyleft JSON file into class-level shared data. + Data is loaded only once and shared across all instances. + + :return: True if successful, False otherwise + """ + if Osadl._data_loaded: + return True + + # OSADL copyleft license checklist from: https://www.osadl.org/Checklists + # Data source: https://www.osadl.org/fileadmin/checklists/copyleft.json + # License: CC-BY-4.0 (Creative Commons Attribution 4.0 International) + # Copyright: (C) 2017 - 2024 Open Source Automation Development Lab (OSADL) eG + try: + f_name = importlib_resources.files(__name__) / 'data/osadl-copyleft.json' + with importlib_resources.as_file(f_name) as f: + with open(f, 'r', encoding='utf-8') as file: + data = json.load(file) + except Exception as e: + self.print_stderr(f'ERROR: Problem loading OSADL copyleft data: {e}') + return False + + # Process copyleft data + copyleft = data.get('copyleft', {}) + if not copyleft: + self.print_stderr('ERROR: No copyleft data found in OSADL JSON') + return False + + # Store in class-level shared dictionary + for lic_id, status in copyleft.items(): + # Normalize license ID (lowercase) for consistent lookup + lic_id_lc = lic_id.lower() + Osadl._shared_copyleft_data[lic_id_lc] = status + + Osadl._data_loaded = True + self.print_debug(f'Loaded {len(Osadl._shared_copyleft_data)} OSADL copyleft entries') + return True + + def is_copyleft(self, spdx_id: str) -> bool: + """ + Check if a license is copyleft according to OSADL data. + + Returns True for both strong copyleft ("Yes") and weak/restricted copyleft ("Yes (restricted)"). + + :param spdx_id: SPDX license identifier + :return: True if copyleft, False otherwise + """ + if not spdx_id: + self.print_debug('No license ID provided for copyleft check') + return False + + # Normalize lookup + spdx_id_lc = spdx_id.lower() + # Use class-level shared data + status = Osadl._shared_copyleft_data.get(spdx_id_lc) + + if not status: + self.print_debug(f'No OSADL copyleft data for license: {spdx_id}') + return False + + # Consider both "Yes" and "Yes (restricted)" as copyleft (case-insensitive) + return status.lower().startswith('yes') + + +# +# End of Osadl Class +# diff --git a/tests/test_osadl.py b/tests/test_osadl.py new file mode 100644 index 00000000..6e0b929f --- /dev/null +++ b/tests/test_osadl.py @@ -0,0 +1,102 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import unittest + +from scanoss.osadl import Osadl + + +class TestOsadl(unittest.TestCase): + """ + Test the Osadl class + """ + + def test_initialization(self): + """Test basic initialization - data is loaded at class level""" + osadl = Osadl() + self.assertIsNotNone(osadl) + self.assertTrue(Osadl._data_loaded) + self.assertGreater(len(Osadl._shared_copyleft_data), 0) + + def test_initialization_with_debug(self): + """Test initialization with debug enabled""" + osadl = Osadl(debug=True) + self.assertTrue(osadl.debug) + + def test_is_copyleft_gpl_2_0_only(self): + """Test GPL-2.0-only is copyleft""" + osadl = Osadl() + self.assertTrue(osadl.is_copyleft('GPL-2.0-only')) + + def test_is_copyleft_gpl_2_0_or_later(self): + """Test GPL-2.0-or-later is copyleft""" + osadl = Osadl() + self.assertTrue(osadl.is_copyleft('GPL-2.0-or-later')) + + def test_is_not_copyleft_mit(self): + """Test MIT is not copyleft""" + osadl = Osadl() + self.assertFalse(osadl.is_copyleft('MIT')) + + def test_is_copyleft_case_insensitive_license_id(self): + """Test license ID lookup is case-insensitive""" + osadl = Osadl() + self.assertTrue(osadl.is_copyleft('gpl-2.0-only')) + self.assertTrue(osadl.is_copyleft('GPL-2.0-ONLY')) + self.assertTrue(osadl.is_copyleft('Gpl-2.0-Only')) + + def test_is_copyleft_unknown_license(self): + """Test unknown license returns False""" + osadl = Osadl() + self.assertFalse(osadl.is_copyleft('Unknown-License')) + + def test_is_copyleft_empty_string(self): + """Test empty string returns False""" + osadl = Osadl() + self.assertFalse(osadl.is_copyleft('')) + + def test_is_copyleft_none(self): + """Test None returns False""" + osadl = Osadl() + self.assertFalse(osadl.is_copyleft(None)) + + def test_multiple_instances_share_data(self): + """Test that multiple instances share the same class-level data""" + osadl1 = Osadl() + osadl2 = Osadl() + + # Both instances should see data loaded by first instance + result1 = osadl1.is_copyleft('GPL-2.0-only') + self.assertTrue(result1) + self.assertTrue(Osadl._data_loaded) + + # Second instance uses the same class-level shared data + result2 = osadl2.is_copyleft('MIT') + self.assertFalse(result2) + + # Verify both instances reference the same class-level data + self.assertIs(Osadl._shared_copyleft_data, Osadl._shared_copyleft_data) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_policy_inspect.py b/tests/test_policy_inspect.py index a3161a45..27c6ef41 100644 --- a/tests/test_policy_inspect.py +++ b/tests/test_policy_inspect.py @@ -579,7 +579,7 @@ def test_copyleft_policy_license_sources_with_markdown_output(self): def test_copyleft_policy_license_sources_with_include_filter(self): """ Test license_sources works with include filter - Filter to scancode source and include only GPL-2.0-or-later + Filter to scancode source and include MIT (normally not copyleft) """ script_dir = os.path.dirname(os.path.abspath(__file__)) file_name = 'result.json' @@ -588,18 +588,19 @@ def test_copyleft_policy_license_sources_with_include_filter(self): filepath=input_file_name, format_type='json', license_sources=['scancode'], - include='GPL-2.0-or-later' + include='MIT' ) status, policy_output = copyleft.run() details = json.loads(policy_output.details) - # Should find only GPL-2.0-or-later from scancode + # Should find MIT (added via include) and any OSADL copyleft licenses self.assertEqual(status, PolicyStatus.POLICY_FAIL.value) - if details: # May be empty if no matches - for component in details.get('components', []): - for license in component['licenses']: - self.assertEqual(license['spdxid'], 'GPL-2.0-or-later') - self.assertEqual(license['source'], 'scancode') + self.assertGreater(len(details.get('components', [])), 0) + + # Verify all licenses are from scancode or unknown (always included) + for component in details.get('components', []): + for license in component['licenses']: + self.assertIn(license['source'], ['scancode', 'unknown']) def test_copyleft_policy_license_sources_with_exclude_filter(self): """ From b9fa952946606ac69cf2e1dc84fd130424f10866 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 17 Nov 2025 13:21:27 +0100 Subject: [PATCH 405/489] chore(linter): improve comment formatting --- src/scanoss/scanner.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 6e5d147b..cde0ad88 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -394,8 +394,8 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, - # add it to the queue + # If the WFP is bigger than the max post size and we already have something + # stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -404,7 +404,8 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 scan_block += wfp scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 + # If the scan request block (group of WFPs) is larger than the POST size + # or we have reached the file limit, add it to the queue if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -413,7 +414,10 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): - self.print_stderr('Warning: Some errors encounted while scanning. Results might be incomplete.') + self.print_stderr( + 'Warning: Some errors encountered while scanning. ' + 'Results might be incomplete.' + ) success = False # End for loop if self.threaded_scan and scan_block != '': @@ -659,7 +663,8 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something stored in the scan block, add it to the queue # noqa: E501 + # If the WFP is bigger than the max post size and we already have something + # stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -668,7 +673,8 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 scan_block += wfp scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 - # If the scan request block (group of WFPs) or larger than the POST size or we have reached the file limit, add it to the queue # noqa: E501 + # If the scan request block (group of WFPs) is larger than the POST size + # or we have reached the file limit, add it to the queue if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: self.threaded_scan.queue_add(scan_block) queue_size += 1 @@ -678,7 +684,8 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 scan_started = True if not self.threaded_scan.run(wait=False): self.print_stderr( - 'Warning: Some errors encounted while scanning. Results might be incomplete.' + 'Warning: Some errors encountered while scanning. ' + 'Results might be incomplete.' ) success = False @@ -799,7 +806,9 @@ def scan_wfp_file(self, file: str = None) -> bool: # noqa: PLR0912, PLR0915 f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' ) if self.debug and cur_size > self.max_post_size: - Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') + Scanner.print_stderr( + f'Warning: Post size {cur_size} greater than limit {self.max_post_size}' + ) scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store if bar: bar.next(batch_files) From a156202cfb575c65b34931ed6cb76f4931319538 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 17 Nov 2025 13:21:55 +0100 Subject: [PATCH 406/489] chore: bump version to 1.41.0 --- CHANGELOG.md | 5 ++++- src/scanoss/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 532ba071..876c5039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Upcoming changes... +## [1.41.0] - 2025-11-17 ### Added - Added `--license-sources` (`-ls`) option to copyleft inspection - Filter which license sources to check (component_declared, license_file, file_header, file_spdx_tag, scancode) @@ -747,4 +749,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.38.0]: https://github.com/scanoss/scanoss.py/compare/v1.37.1...v1.38.0 [1.39.0]: https://github.com/scanoss/scanoss.py/compare/v1.38.0...v1.39.0 [1.40.0]: https://github.com/scanoss/scanoss.py/compare/v1.39.0...v1.40.0 -[1.40.1]: https://github.com/scanoss/scanoss.py/compare/v1.40.0...v1.40.1 \ No newline at end of file +[1.40.1]: https://github.com/scanoss/scanoss.py/compare/v1.40.0...v1.40.1 +[1.41.0]: https://github.com/scanoss/scanoss.py/compare/v1.40.1...v1.41.0 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 694457df..46c39ba1 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.40.1' +__version__ = '1.41.0' From a764b6f3f74f1ec93455511db9e707f6f6c1d4b5 Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Wed, 17 Dec 2025 14:03:23 +0100 Subject: [PATCH 407/489] fix(hfh): use new components request in hfh scan (#173) --- CHANGELOG.md | 6 +++++- src/scanoss/__init__.py | 2 +- src/scanoss/scanners/scanner_hfh.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 876c5039..f9ce3520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... +## [1.41.1] - 2025-11-17 +- Use `components` instead of `purls` for vulnerability detection when converting to CycloneDX format in Folder Scan + ## [1.41.0] - 2025-11-17 ### Added - Added `--license-sources` (`-ls`) option to copyleft inspection @@ -750,4 +753,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.39.0]: https://github.com/scanoss/scanoss.py/compare/v1.38.0...v1.39.0 [1.40.0]: https://github.com/scanoss/scanoss.py/compare/v1.39.0...v1.40.0 [1.40.1]: https://github.com/scanoss/scanoss.py/compare/v1.40.0...v1.40.1 -[1.41.0]: https://github.com/scanoss/scanoss.py/compare/v1.40.1...v1.41.0 \ No newline at end of file +[1.41.0]: https://github.com/scanoss/scanoss.py/compare/v1.40.1...v1.41.0 +[1.41.1]: https://github.com/scanoss/scanoss.py/compare/v1.41.0...v1.41.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 46c39ba1..61da18d9 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.41.0' +__version__ = '1.41.1' diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 8d4a2849..739a8921 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -218,7 +218,7 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 } get_vulnerabilities_json_request = { - 'purls': [{'purl': purl, 'requirement': best_match_version['version']}], + 'components': [{'purl': purl, 'requirement': best_match_version['version']}], } decorated_scan_results = self.scanner.client.get_dependencies(get_dependencies_json_request) From 1e0f941918444d1abbfacb6a0c8e8bbf60eaa4d8 Mon Sep 17 00:00:00 2001 From: Mariano Scasso <75589700+mscasso-scanoss@users.noreply.github.com> Date: Thu, 18 Dec 2025 09:05:20 -0300 Subject: [PATCH 408/489] Feature/mscasso/sp 3430 add line headers filter (#172) * Add file headers filter feature. * Add HeaderFilter class. * Add --skip-headers and --skip-headers-limit flags. * Update unit tests and workflows. * Update CHANGELOG. --------- Co-authored-by: eeisegn --- .github/workflows/container-local-test.yml | 8 +- .github/workflows/container-publish-ghcr.yml | 7 + .github/workflows/lint.yml | 31 +- .github/workflows/python-local-test.yml | 16 +- .github/workflows/python-publish-pypi.yml | 25 + CHANGELOG.md | 9 +- Makefile | 18 +- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 17 + src/scanoss/header_filter.py | 563 +++++++++++++++++++ src/scanoss/scanner.py | 5 + src/scanoss/scanossapi.py | 2 +- src/scanoss/scanossbase.py | 2 +- src/scanoss/winnowing.py | 90 ++- tests/test_headers_filter.py | 370 ++++++++++++ tests/test_winnowing.py | 146 ++++- tools/linter.sh | 46 +- 17 files changed, 1280 insertions(+), 77 deletions(-) create mode 100644 src/scanoss/header_filter.py create mode 100644 tests/test_headers_filter.py diff --git a/.github/workflows/container-local-test.yml b/.github/workflows/container-local-test.yml index 447e6053..053f794b 100644 --- a/.github/workflows/container-local-test.yml +++ b/.github/workflows/container-local-test.yml @@ -98,4 +98,10 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi - + docker run -v "$(pwd)":"/scanoss" ${{ env.IMAGE_NAME }} wfp --skip-headers -o fingers.wfp tests + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/container-publish-ghcr.yml b/.github/workflows/container-publish-ghcr.yml index 3d8fb39d..84aebab0 100644 --- a/.github/workflows/container-publish-ghcr.yml +++ b/.github/workflows/container-publish-ghcr.yml @@ -137,6 +137,13 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + docker run -v "$(pwd)":"/scanoss" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} wfp --skip-headers -o fingers.wfp tests + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi # Install the cosign tool except on PR # - name: Install cosign diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 842d6f76..11367ddc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,35 +24,6 @@ jobs: python -m pip install --upgrade pip pip install ruff - - name: Get changed Python files - id: changed_files - run: | - # Find the merge base between the main branch and the current HEAD. - merge_base=$(git merge-base origin/main HEAD) - # List all changed Python files since the merge base. - files=$(git diff --name-only "$merge_base" HEAD | grep '\.py$' || true) - - # Filter out files that match exclude patterns from pyproject.toml - # this is a temporary workaround until we fix all the lint errors - filtered_files=$(echo "$files" | grep -v -E 'tests/|test_.*\.py|src/protoc_gen_swagger/|src/scanoss/api/' || true) - - # Use the multi-line syntax for outputs. - echo "files<> "$GITHUB_OUTPUT" - echo "${filtered_files}" >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - - echo "Changed files before filtering: ${files}" - echo "Changed files after filtering: ${filtered_files}" - - name: Run Ruff on changed files run: | - if [ -z "${{ steps.changed_files.outputs.files }}" ]; then - echo "No Python files changed. Exiting." - exit 0 - else - echo "Linting the following files:" - echo "${{ steps.changed_files.outputs.files }}" - # Pass the list of changed files to Ruff. - echo "${{ steps.changed_files.outputs.files }}" | xargs ruff check - fi - + make lint diff --git a/.github/workflows/python-local-test.yml b/.github/workflows/python-local-test.yml index 50e13960..59aa6813 100644 --- a/.github/workflows/python-local-test.yml +++ b/.github/workflows/python-local-test.yml @@ -60,6 +60,13 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + scanoss-py wfp --skip-headers tests > fingers.wfp + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi - name: Run Tests (fast winnowing) run: | @@ -74,7 +81,13 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi - + scanoss-py wfp --skip-headers tests > fingers.wfp + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi - name: Run Tests HPSM (fast winnowing) run: | @@ -89,7 +102,6 @@ jobs: echo "Error: WFP test did not produce any results. Failing" exit 1 fi - - name: Run Unit Tests run: | diff --git a/.github/workflows/python-publish-pypi.yml b/.github/workflows/python-publish-pypi.yml index 2838bed9..1dc8b571 100644 --- a/.github/workflows/python-publish-pypi.yml +++ b/.github/workflows/python-publish-pypi.yml @@ -35,6 +35,10 @@ jobs: pip install dist/scanoss-*-py3-none-any.whl which scanoss-py + - name: Run Unit Tests + run: | + make unit_test + - name: Run Local Tests run: | which scanoss-py @@ -47,6 +51,13 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + scanoss-py wfp --skip-headers tests > fingers.wfp + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi pip uninstall -y scanoss - name: Publish Package - ${{ github.ref_name }} @@ -102,6 +113,13 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + scanoss-py wfp --skip-headers tests > fingers.wfp + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi - name: Run Tests (fast winnowing) run: | @@ -116,4 +134,11 @@ jobs: echo "Error: Scan test did not produce any results. Failing" exit 1 fi + scanoss-py wfp --skip-headers tests > fingers.wfp + wfp_count=$(cat fingers.wfp | grep 'file=' | wc -l) + echo "WFP Count: $wfp_count" + if [[ $wfp_count -lt 1 ]]; then + echo "Error: WFP test did not produce any results. Failing" + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ce3520..4a89cf72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... -## [1.41.1] - 2025-11-17 +## [1.42.0] - 2025-12-17 +### Added +- Added support for filtering uninteresting data from the beginning of source files. + - When using `--skip-headers` it will skip over copyright notices, import statements, comments, etc. + - The `--skip-headers-limit` option specifies the maximum number of lines to skip if required. + +## [1.41.1] - 2025-12-16 - Use `components` instead of `purls` for vulnerability detection when converting to CycloneDX format in Folder Scan ## [1.41.0] - 2025-11-17 @@ -755,3 +761,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.40.1]: https://github.com/scanoss/scanoss.py/compare/v1.40.0...v1.40.1 [1.41.0]: https://github.com/scanoss/scanoss.py/compare/v1.40.1...v1.41.0 [1.41.1]: https://github.com/scanoss/scanoss.py/compare/v1.41.0...v1.41.1 +[1.42.0]: https://github.com/scanoss/scanoss.py/compare/v1.41.1...v1.42.0 \ No newline at end of file diff --git a/Makefile b/Makefile index 9c13989a..95af3519 100644 --- a/Makefile +++ b/Makefile @@ -53,18 +53,28 @@ publish_test: ## Publish the Python package to TestPyPI @echo "Publishing package to TestPyPI..." twine upload --repository testpypi dist/* -lint-docker: ## Run ruff linter with docker +unit_test: ## Run unit tests + @echo "Running unit tests..." + @python -m unittest + +lint-docker: ## Run ruff linter with docker @./tools/linter.sh --docker -lint-docker-fix: ## Run ruff linter with docker and auto-fix +lint-docker-fix: ## Run ruff linter with docker and auto-fix @./tools/linter.sh --docker --fix -lint: ## Run ruff linter locally +lint: ## Run ruff linter locally @./tools/linter.sh -lint-fix: ## Run ruff linter locally with auto-fix +lint-fix: ## Run ruff linter locally with auto-fix @./tools/linter.sh --fix +lint-all: ## Run ruff linter locally for all files + @./tools/linter.sh --all + +lint-fix-all: ## Run ruff linter locally with auto-fix for all files + @./tools/linter.sh --fix --all + publish: ## Publish Python package to PyPI @echo "Publishing package to PyPI..." twine upload dist/* diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 61da18d9..224053eb 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.41.1' +__version__ = '1.42.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 0a3c1ae0..d38b2517 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1096,6 +1096,19 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p.add_argument('--skip-md5', '-5', type=str, action='append', help='Skip files matching MD5.') p.add_argument('--strip-hpsm', '-G', type=str, action='append', help='Strip HPSM string from WFP.') p.add_argument('--strip-snippet', '-N', type=str, action='append', help='Strip Snippet ID string from WFP.') + p.add_argument( + '--skip-headers', + '-skh', + action='store_true', + help='Skip license headers, comments and imports at the beginning of files.', + ) + p.add_argument( + '--skip-headers-limit', + '-shl', + type=int, + default=0, + help='Maximum number of lines to skip when filtering headers (default: 0 = no limit).', + ) # Global Scan/GRPC options for p in [ @@ -1388,6 +1401,8 @@ def wfp(parser, args): strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, scan_settings=scan_settings, + skip_headers=args.skip_headers, + skip_headers_limit=args.skip_headers_limit, ) if args.stdin: contents = sys.stdin.buffer.read() @@ -1583,6 +1598,8 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 scan_settings=scan_settings, req_headers=process_req_headers(args.header), use_grpc=args.grpc, + skip_headers=args.skip_headers, + skip_headers_limit=args.skip_headers_limit, ) if args.wfp: if not scanner.is_file_or_snippet_scan(): diff --git a/src/scanoss/header_filter.py b/src/scanoss/header_filter.py new file mode 100644 index 00000000..d6fe10fe --- /dev/null +++ b/src/scanoss/header_filter.py @@ -0,0 +1,563 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + Line Filter Module - Identifies where real source code implementation begins. + + This module analyzes source code files and determines which lines are: + - License headers + - Documentation comments + - Imports/includes + - Blank lines + + And returns the content from where the real implementation begins. +""" + +import re +from pathlib import Path +from typing import Optional, Tuple + +from .scanossbase import ScanossBase + + +class LanguagePatterns: + """ + Regex patterns for different programming languages. + + This class provides a collection of regex patterns for identifying different + programming constructs, handling imports, comments, and license statements + across various programming languages. The main purpose of this class is to + assist in parsing or analysing code written in different languages efficiently. + + :ivar COMMENT_PATTERNS: A dictionary containing regex patterns to identify + single-line and multi-line comments in various programming languages. + :ivar IMPORT_PATTERNS: A dictionary mapping programming languages to their + respective regex patterns for identifying import statements or package + includes it. + :ivar LICENSE_KEYWORDS: A list of keywords commonly found in license texts + or statements, often used to detect the presence of licensing information. + """ + # Comment patterns (single-line and multi-line start/end) + COMMENT_PATTERNS = { + # C-style languages: C, C++, Java, JavaScript, TypeScript, Go, + # Rust, C#, PHP, Kotlin, Scala, Dart, Objective-C + 'c_style': { + 'single_line': r'^\s*//.*$', + 'multi_start': r'^\s*/\*', + 'multi_end': r'\*/\s*$', + 'multi_single': r'^\s*/\*.*\*/\s*$', + }, + # Python, shell scripts, Ruby, Perl, R, Julia, YAML + 'python_style': { + 'single_line': r'^\s*#.*$', + 'doc_string_start': r'^\s*"""', + 'doc_string_end': r'"""\s*$', + }, + # Lua, SQL, Haskell + 'lua_style': { + 'single_line': r'^\s*--.*$', + 'multi_start': r'^\s*--\[\[', + 'multi_end': r'\]\]\s*$', + }, + # HTML, XML + 'html_style': { + 'multi_start': r'^\s*\s*$', + 'multi_single': r'^\s*\s*$', + }, + } + # Import/include patterns by language + IMPORT_PATTERNS = { + 'python': [ + r'^\s*import\s+', + r'^\s*from\s+.*\s+import\s+', + ], + 'javascript': [ + r'^\s*import\s+.*\s+from\s+', + r'^\s*import\s+["\']', + r'^\s*import\s+type\s+', + r'^\s*export\s+\*\s+from\s+', + r'^\s*export\s+\{.*\}\s+from\s+', + r'^\s*const\s+.*\s*=\s*require\(', + r'^\s*var\s+.*\s*=\s*require\(', + r'^\s*let\s+.*\s*=\s*require\(', + ], + 'typescript': [ + r'^\s*import\s+', + r'^\s*export\s+.*\s+from\s+', + r'^\s*import\s+type\s+', + r'^\s*import\s+\{.*\}\s+from\s+', + ], + 'java': [ + r'^\s*import\s+', + r'^\s*package\s+', + ], + 'kotlin': [ + r'^\s*import\s+', + r'^\s*package\s+', + ], + 'scala': [ + r'^\s*import\s+', + r'^\s*package\s+', + ], + 'go': [ + r'^\s*import\s+\(', + r'^\s*import\s+"', + r'^\s*package\s+', + r'^\s*"[^"]*"\s*$', # Imports inside import () block + # Imports with alias: name "package" + r'^\s*[a-zA-Z_][a-zA-Z0-9_]*\s+"[^"]*"\s*$', + r'^\s*_\s+"[^"]*"\s*$', # _ "package" imports + ], + 'rust': [ + r'^\s*use\s+', + r'^\s*extern\s+crate\s+', + r'^\s*mod\s+', + ], + 'cpp': [ + r'^\s*#include\s+', + r'^\s*#pragma\s+', + r'^\s*#ifndef\s+.*_H.*', # Header guards: #ifndef FOO_H + r'^\s*#define\s+.*_H.*', # Header guards: #define FOO_H + # #endif at end of file (may have comment) + r'^\s*#endif\s+(//.*)?\s*$', + ], + 'csharp': [ + r'^\s*using\s+', + r'^\s*namespace\s+', + ], + 'php': [ + r'^\s*use\s+', + r'^\s*require\s+', + r'^\s*require_once\s+', + r'^\s*include\s+', + r'^\s*include_once\s+', + r'^\s*namespace\s+', + ], + 'swift': [ + r'^\s*import\s+', + ], + 'ruby': [ + r'^\s*require\s+', + r'^\s*require_relative\s+', + r'^\s*load\s+', + ], + 'perl': [ + r'^\s*use\s+', + r'^\s*require\s+', + ], + 'r': [ + r'^\s*library\(', + r'^\s*require\(', + r'^\s*source\(', + ], + 'lua': [ + r'^\s*require\s+', + r'^\s*local\s+.*\s*=\s*require\(', + ], + 'dart': [ + r'^\s*import\s+', + r'^\s*export\s+', + r'^\s*part\s+', + ], + 'haskell': [ + r'^\s*import\s+', + r'^\s*module\s+', + ], + 'elixir': [ + r'^\s*import\s+', + r'^\s*alias\s+', + r'^\s*require\s+', + r'^\s*use\s+', + ], + 'clojure': [ + r'^\s*\(\s*ns\s+', + r'^\s*\(\s*require\s+', + r'^\s*\(\s*import\s+', + ], + } + # Keywords that indicate licenses + LICENSE_KEYWORDS = [ + 'copyright', 'license', 'licensed', 'all rights reserved', + 'permission', 'redistribution', 'warranty', 'liability', + 'apache', 'mit', 'gpl', 'bsd', 'mozilla', 'author:', + 'spdx-license', 'contributors', 'licensee' + ] + +COMPLETE_DOCSTRING_QUOTE_COUNT = 2 +LICENSE_HEADER_MAX_LINES = 50 +# Map of file extensions to programming languages +EXT_MAP = { + '.py': 'python', + '.js': 'javascript', + '.mjs': 'javascript', + '.cjs': 'javascript', + '.ts': 'typescript', + '.tsx': 'typescript', + '.jsx': 'javascript', + '.java': 'java', + '.kt': 'kotlin', + '.kts': 'kotlin', + '.scala': 'scala', + '.sc': 'scala', + '.go': 'go', + '.rs': 'rust', + '.cpp': 'cpp', + '.cc': 'cpp', + '.cxx': 'cpp', + '.c': 'cpp', + '.h': 'cpp', + '.hpp': 'cpp', + '.hxx': 'cpp', + '.cs': 'csharp', + '.php': 'php', + '.swift': 'swift', + '.rb': 'ruby', + '.pl': 'perl', + '.pm': 'perl', + '.r': 'r', + '.R': 'r', + '.lua': 'lua', + '.dart': 'dart', + '.hs': 'haskell', + '.ex': 'elixir', + '.exs': 'elixir', + '.clj': 'clojure', + '.cljs': 'clojure', + '.m': 'cpp', # Objective-C + '.mm': 'cpp', # Objective-C++ + # Shell scripts share Python's # comment style, but lack dedicated + # import patterns (source/. commands won't be filtered) + '.sh': 'python', + '.bash': 'python', + '.zsh': 'python', + '.fish': 'python', +} + + +def is_blank_line(stripped_line: str) -> bool: + """ + Check if a line is blank. + + This method determines whether a given string `line` is blank by checking + if it consists entirely of whitespace or is empty. + + :param stripped_line: The string to be evaluated. + :return: True if the string is blank, otherwise False. + """ + return len(stripped_line) == 0 + + +def is_shebang(stripped_line: str) -> bool: + """ + Check if the given line is a shebang line. + + This function determines if the provided string is a shebang line, + which indicates the path to the interpreter that should execute the + script. + + :param stripped_line: The string to check if it's a shebang line. + :return: True if the given line starts with '#!', otherwise False. + """ + return stripped_line.startswith('#!') + + +class HeaderFilter(ScanossBase): + """ + Source code file analyser that filters headers, comments, and imports. + + This class processes code files and returns only the real + implementation content, omitting licenses, documentation comments, + and imports. + """ + + def __init__( + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + skip_limit: Optional[int] = None + ): + """ + Initialise HeaderFilter + Parameters + ---------- + skip_limit: int + Maximum number of lines to skip when analysing a file. + If set, then stop stripping data after this number of lines. + (None/0 = unlimited by default) + """ + super().__init__(debug, trace, quiet) + self.patterns = LanguagePatterns() + self.max_lines = skip_limit + + def filter(self, file: str, decoded_contents: str) -> int: + """ + Main method that filters file content + Parameters + ---------- + :param file: File path (used to detect extension) + :param decoded_contents: File contents in utf-8 encoding + Return + ------ + - line_offset: Number of lines skipped from the beginning + (0 if no filtering) + """ + if not decoded_contents or not file: + self.print_msg(f'No file or contents provided, skipping line filter for: {file}') + return 0 + self.print_debug(f'HeaderFilter processing file: {file}') + # Detect language + language = self.detect_language(file) + # If language is not supported, return original content + if not language: + self.print_debug(f'Skipping line filter for unsupported language: {file}') + return 0 + lines = decoded_contents.splitlines(keepends=True) + num_lines = len(lines) + if num_lines == 0: + self.print_msg(f'No lines in file: {file}') + return 0 + self.print_debug(f'Analysing {num_lines} lines for file: {file}') + + # Find the first implementation line (optimised - stops at first match) + implementation_start = self.find_first_implementation_line(lines, language) + # If no implementation, return empty + if implementation_start is None: + self.print_debug(f'No implementation found in file: {file}') + return 0 + # Calculate how many lines were filtered out (line_offset) + line_offset = implementation_start - 1 + # Apply max_lines limit if configured + if self.max_lines is not None and 0 < self.max_lines < line_offset: + self.print_trace( + f'Line offset {line_offset} exceeds max_lines {self.max_lines}, ' + f'capping at {self.max_lines} for: {file}' + ) + line_offset = self.max_lines + + if line_offset > 0: + self.print_debug(f'Filtered out {line_offset} lines from beginning of {file} (language: {language})') + return line_offset + + def detect_language(self, file_path: str) -> Optional[str]: + """ + Detects the programming language based on the provided file extension. + + This function uses a predefined mapping between file extensions and programming + languages to determine the language associated with the file. If the file extension + is found in the mapping, the corresponding language is returned. Otherwise, it + returns None. + + :param file_path: Path to the file whose programming language needs to be detected. + :return: The programming language corresponding to the file extension if mapped, + otherwise None. + """ + path = Path(file_path) + extension = path.suffix.lower() + if extension: + detected_language = EXT_MAP.get(extension) + if detected_language: + self.print_debug(f'Detected language "{detected_language}" for extension "{extension}"') + else: + self.print_debug(f'No language mapping found for extension "{extension}"') + else: + self.print_debug(f'No file extension found, skipping language detection for: {file_path}') + detected_language = None + return detected_language + + def is_license_header(self, line: str) -> bool: + """ + Check if the line appears to be part of a license header. + + This method evaluates a given line of text to determine whether it + contains keywords that suggest it is part of a license header. It + performs a case-insensitive check against a predefined set of license + keywords. + + :param line: The line of text to check. + :return: True if the line contains keywords indicating it is part of a + license header; False otherwise. + """ + line_lower = line.lower() + return any(keyword in line_lower for keyword in self.patterns.LICENSE_KEYWORDS) + + def get_comment_style(self, language: str) -> str: + """ + Return the comment style associated with a given programming language. + + This method determines the appropriate comment style to use based on the + specified programming language. Supported languages include those with C-style + comments, Python-style comments, and Lua-style comments. If the language does + not match any of the explicitly defined groups, a default of `c_style` is + returned. + + :param language: The name of the programming language for which the comment + style needs to be determined. + :return: The comment style for the provided programming language. Possible + values are 'c_style', 'python_style', or 'lua_style'. + """ + if language: + if language in ['cpp', 'java', 'kotlin', 'scala', 'javascript', 'typescript', + 'go', 'rust', 'csharp', 'php', 'swift', 'dart']: + return 'c_style' + if language in ['python', 'ruby', 'perl', 'r']: + return 'python_style' + if language in ['lua', 'haskell']: + return 'lua_style' + self.print_debug(f'No comment style defined for language "{language}", using default: "c_style"') + return 'c_style' # Default + + def is_comment(self, line: str, in_multiline: bool, patterns: dict) -> Tuple[bool, bool]: # noqa: PLR0911 + """ + Check if a line is a comment + + :param patterns: comment patterns + :param line: Line to check + :param in_multiline: Whether we're currently in a multiline comment + :return: Tuple of (is_comment, still_in_multiline) + """ + if not patterns: + self.print_msg('No comment patterns defined, skipping comment check') + return False, in_multiline + # If we're in a multiline comment + if in_multiline: + # Check if the comment ends + if 'multi_end' in patterns and re.search(patterns['multi_end'], line): + return True, False + if 'doc_string_end' in patterns and re.search(patterns['doc_string_end'], line): + return True, False + return True, True + # Single-line comment + if 'single_line' in patterns and re.match(patterns['single_line'], line): + return True, False + # Multiline comment complete in one line + if 'multi_single' in patterns and re.match(patterns['multi_single'], line): + return True, False + # Start of multiline comment (C-style) + if 'multi_start' in patterns and re.search(patterns['multi_start'], line): + # If it also ends on the same line + if 'multi_end' in patterns and re.search(patterns['multi_end'], line): + return True, False + return True, True + # Start of docstring (Python) + if 'doc_string_start' in patterns and '"""' in line: + # Count how many quotes there are + count = line.count('"""') + if count == COMPLETE_DOCSTRING_QUOTE_COUNT: # Complete docstring in one line + return True, False + if count == 1: # Start of a multiline docstring + return True, True + # Default response: not a comment + return False, in_multiline + + def is_import(self, line: str, patterns: dict) -> bool: + """ + Check if a line of code is an import or include statement for a given programming language. + + This function determines whether a specific line of code matches any + import/include patterns defined for the provided programming language. + It relies on predefined regular expression patterns. + + :param patterns: import patterns for the given language. + :param line: A single line of code to check. + :return: True if the line matches any import/include pattern for the given language, + otherwise False. + """ + if not patterns: + self.print_debug('No import patterns defined, skipping import check') + return any(re.match(pattern, line) for pattern in patterns) + + def find_first_implementation_line(self, lines: list[str], language: str) -> Optional[int]: # noqa: PLR0912 + """ + Find the line number where the implementation begins (optimised version). + Returns as soon as the first implementation line is found. + + :param lines: List of code lines + :param language: Programming language + :return: Line number (1-indexed) where implementation starts, or None if not found + """ + if not lines or not language: + self.print_debug('No lines or language provided, skipping implementation line detection') + return None + in_multiline_comment = False + in_license_section = False + in_import_block = False # To handle import blocks in Go + consecutive_imports_count = 0 + # Get comment & import patterns for the language + comment_patterns = self.patterns.COMMENT_PATTERNS[self.get_comment_style(language)] + import_patterns = self.patterns.IMPORT_PATTERNS[language] + # Iterate through lines trying to find the first implementation line + for i, line in enumerate(lines): + line_number = i + 1 + stripped = line.strip() + # Shebang (only first line) or blank line + if (i == 0 and is_shebang(stripped)) or is_blank_line(stripped): + continue + # Check if it's a comment + is_a_comment, in_multiline_comment = self.is_comment(line, in_multiline_comment, comment_patterns) + if is_a_comment: + # Check if it's part of the license header + if self.is_license_header(line): + if not in_license_section: + self.print_trace(f'Line {line_number}: Detected license header section') + in_license_section = True + # If still in the license section (first lines) + elif in_license_section and line_number < LICENSE_HEADER_MAX_LINES: + pass # Still in the license section. Keep looking. + else: + if in_license_section: + self.print_trace(f'Line {line_number}: End of license header section') + in_license_section = False + continue + # If not a comment but we find a non-empty line, end license section + if not is_a_comment: + in_license_section = False + # Handle import blocks in Go + if language == 'go': + if stripped.startswith('import ('): + self.print_trace(f'Line {line_number}: Detected Go import block start') + in_import_block = True + continue + if in_import_block: + if stripped == ')': + self.print_trace(f'Line {line_number}: Detected Go import block end') + in_import_block = False + continue + if (stripped.startswith('"') or stripped.startswith('_') or + re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\s+"', stripped)): + # It's part of the import block + continue + # Check if it's an import + if self.is_import(line, import_patterns): + if consecutive_imports_count == 0: + self.print_trace(f'Line {line_number}: Detected import section') + consecutive_imports_count += 1 + continue + # If we get here, it's implementation code - return immediately! + self.print_trace(f'Line {line_number}: First implementation line detected') + return line_number + # End for loop? + return None +# +# End of HeaderFilter Class +# \ No newline at end of file diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index cde0ad88..cec80326 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -109,6 +109,8 @@ def __init__( # noqa: PLR0913, PLR0915 scan_settings: 'ScanossSettings | None' = None, req_headers: dict = None, use_grpc: bool = False, + skip_headers: bool = False, + skip_headers_limit: int = 0, ): """ Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning @@ -137,6 +139,7 @@ def __init__( # noqa: PLR0913, PLR0915 self.winnowing = Winnowing( debug=debug, + trace=trace, quiet=quiet, skip_snippets=self._skip_snippets, all_extensions=all_extensions, @@ -145,6 +148,8 @@ def __init__( # noqa: PLR0913, PLR0915 strip_hpsm_ids=strip_hpsm_ids, strip_snippet_ids=strip_snippet_ids, skip_md5_ids=skip_md5_ids, + skip_headers=skip_headers, + skip_headers_limit=skip_headers_limit, ) self.scanoss_api = ScanossApi( debug=debug, diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index c698a55f..f077585b 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -78,7 +78,7 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 :param api_key: API Key (default None) :param debug: Enable debug (default False) :param trace: Enable trace (default False) - :param quiet: Enable quite mode (default False) + :param quiet: Enable quiet mode (default False) To set a custom certificate use: REQUESTS_CA_BUNDLE=/path/to/cert.pem diff --git a/src/scanoss/scanossbase.py b/src/scanoss/scanossbase.py index de8358ae..07409af0 100644 --- a/src/scanoss/scanossbase.py +++ b/src/scanoss/scanossbase.py @@ -50,7 +50,7 @@ def print_stderr(*args, **kwargs): def print_msg(self, *args, **kwargs): """ - Print message if quite mode is not enabled + Print message if quiet mode is not enabled """ if not self.quiet: self.print_stderr(*args, **kwargs) diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 9b2e3a57..08588cb5 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -37,6 +37,7 @@ from binaryornot.check import is_binary from crc32c import crc32c +from .header_filter import HeaderFilter from .scanossbase import ScanossBase # Winnowing configuration. DO NOT CHANGE. @@ -172,6 +173,8 @@ def __init__( # noqa: PLR0913 strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None, + skip_headers: bool = False, + skip_headers_limit: int = 0, ): """ Instantiate Winnowing class @@ -198,7 +201,9 @@ def __init__( # noqa: PLR0913 self.strip_hpsm_ids = strip_hpsm_ids self.strip_snippet_ids = strip_snippet_ids self.hpsm = hpsm + self.skip_headers = skip_headers self.is_windows = platform.system() == 'Windows' + self.header_filter = HeaderFilter(debug=debug, trace=trace, quiet=quiet, skip_limit=skip_headers_limit) if hpsm: self.crc8_maxim_dow_table = [] self.crc8_generate_table() @@ -353,6 +358,48 @@ def __strip_snippets(self, file: str, wfp: str) -> str: self.print_debug(f'Stripped snippet ids from {file}') return wfp + def __strip_lines_until_offset(self, file: str, wfp: str, line_offset: int) -> str: + """ + Strip lines from the WFP up to and including the line_offset + + :param file: name of fingerprinted file + :param wfp: WFP to clean + :param line_offset: line number offset to strip up to + :return: updated WFP + """ + # No offset specified, return original WFP + if line_offset <= 0: + return wfp + lines = wfp.split('\n') + filtered_lines = [] + start_line_added = False + for line in lines: + # Check if a line contains snippet data (format: line_number=hash,hash,...) + line_details = line.split('=') + if line_details[0].isdigit(): + try: + line_num = int(line_details[0]) + # Keep lines that are after the offset + # (line_offset is the last line previous to real code) + if line_num > line_offset: + # Add the start_line tag before the first snippet line + if not start_line_added: + filtered_lines.append(f'start_line={line_offset}') + start_line_added = True + filtered_lines.append(line) + except (ValueError, IndexError) as e: + self.print_stderr(f'Error decoding line number from line {line} in {file}: {e}') + # Keep non-snippet lines (like file=, hpsm=, etc.) + filtered_lines.append(line) + else: + # Keep non-snippet lines (like file=, hpsm=, etc.) + filtered_lines.append(line) + # End for loop comment + wfp = '\n'.join(filtered_lines) + if start_line_added: + self.print_debug(f'Stripped lines up to offset {line_offset} from {file}') + return wfp + def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool]: """Detect the types of line endings present in file contents. @@ -362,13 +409,14 @@ def __detect_line_endings(self, contents: bytes) -> Tuple[bool, bool, bool]: Returns: Tuple of (has_crlf, has_lf_only, has_cr_only, has_mixed) indicating which line ending types are present. """ + if not contents: + self.print_debug('Warning: No file contents provided') has_crlf = b'\r\n' in contents # For LF detection, we need to find LF that's not part of CRLF content_without_crlf = contents.replace(b'\r\n', b'') has_standalone_lf = b'\n' in content_without_crlf # For CR detection, we need to find CR that's not part of CRLF has_standalone_cr = b'\r' in content_without_crlf - return has_crlf, has_standalone_lf, has_standalone_cr def __calculate_opposite_line_ending_hash(self, contents: bytes): @@ -384,13 +432,11 @@ def __calculate_opposite_line_ending_hash(self, contents: bytes): Hash with opposite line endings as hex string, or None if no line endings detected. """ has_crlf, has_standalone_lf, has_standalone_cr = self.__detect_line_endings(contents) - if not has_crlf and not has_standalone_lf and not has_standalone_cr: + self.print_debug('No line endings detected in file contents') return None - - # Normalize all line endings to LF first + # Normalise all line endings to LF first normalized = contents.replace(b'\r\n', b'\n').replace(b'\r', b'\n') - # Determine the dominant line ending type if has_crlf and not has_standalone_lf and not has_standalone_cr: # File is Windows (CRLF) - produce Unix (LF) hash @@ -398,7 +444,7 @@ def __calculate_opposite_line_ending_hash(self, contents: bytes): else: # File is Unix (LF/CR) or mixed - produce Windows (CRLF) hash opposite_contents = normalized.replace(b'\n', b'\r\n') - + # Return the MD5 hash of the opposite contents return hashlib.md5(opposite_contents).hexdigest() def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # noqa: PLR0912, PLR0915 @@ -420,27 +466,26 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # Print file line content_length = len(contents) original_filename = file - if self.is_windows: original_filename = file.replace('\\', '/') wfp_filename = repr(original_filename).strip("'") # return a utf-8 compatible version of the filename - if self.obfuscate: # hide the real size of the file and its name, but keep the suffix + # hide the real size of the file and its name but keep the suffix + if self.obfuscate: wfp_filename = f'{self.ob_count}{pathlib.Path(original_filename).suffix}' self.ob_count = self.ob_count + 1 self.file_map[wfp_filename] = original_filename # Save the file name map for later (reverse lookup) - + # Construct the WFP header wfp = 'file={0},{1},{2}\n'.format(file_md5, content_length, wfp_filename) - - # Add opposite line ending hash based on line ending analysis + # Add the opposite line ending hash based on line ending analysis if not bin_file: opposite_hash = self.__calculate_opposite_line_ending_hash(contents) if opposite_hash is not None: wfp += f'fh2={opposite_hash}\n' - # We don't process snippets for binaries, or other uninteresting files, or if we're requested to skip - if bin_file or self.skip_snippets or self.__skip_snippets(file, contents.decode('utf-8', 'ignore')): + decoded_contents = contents.decode('utf-8', 'ignore') + if bin_file or self.skip_snippets or self.__skip_snippets(file, decoded_contents): return wfp - # Add HPSM + # Add HPSM (calculated from original contents, not filtered) if self.hpsm: hpsm = self.__strip_hpsm(file, self.calc_hpsm(contents)) if len(hpsm) > 0: @@ -448,7 +493,7 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: # Initialize variables gram = '' window = [] - line = 1 + line = 1 # Line counter for WFP generation last_hash = MAX_CRC32 last_line = 0 output = '' @@ -503,12 +548,19 @@ def wfp_for_contents(self, file: str, bin_file: bool, contents: bytes) -> str: wfp += output + '\n' else: self.print_debug(f'Warning: skipping output in WFP for {file} - "{output}"') - + # Warn if we don't have any WFP content if wfp is None or wfp == '': self.print_stderr(f'Warning: No WFP content data for {file}') - elif self.strip_snippet_ids: - wfp = self.__strip_snippets(file, wfp) - + else: + # Apply line filter to remove headers, comments, and imports from the beginning (if enabled) + if self.skip_headers: + line_offset = self.header_filter.filter(file, decoded_contents) + if line_offset > 0: + wfp = self.__strip_lines_until_offset(file, wfp, line_offset) + # Strip snippet IDs from the WFP (if enabled) + if self.strip_snippet_ids: + wfp = self.__strip_snippets(file, wfp) + # Return the WFP contents return wfp def calc_hpsm(self, content): diff --git a/tests/test_headers_filter.py b/tests/test_headers_filter.py new file mode 100644 index 00000000..72b2d888 --- /dev/null +++ b/tests/test_headers_filter.py @@ -0,0 +1,370 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" +import unittest + +from scanoss.header_filter import HeaderFilter + + +class TestHeaderFilter(unittest.TestCase): + """ + Test suite for HeaderFilter class functionality + """ + + def setUp(self): + """Set up test fixtures""" + self.line_filter = HeaderFilter(debug=False, quiet=True) + + def test_python_basic_filtering(self): + """Test basic Python file filtering with license and imports""" + test_content = b"""# Copyright 2024 +# Licensed under MIT +# All rights reserved + +import os +import sys +from pathlib import Path + +def main(): + print('Hello World') + return 0 + +if __name__ == '__main__': + main() +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.py', test_string) + + msg = "Should skip 8 lines (3 license + 1 blank + 3 imports + 1 blank)" + self.assertEqual(line_offset, 8, msg) + + def test_javascript_multiline_comment(self): + """Test JavaScript file with multiline license comment""" + test_content = b"""/* + * Copyright 2024 + * Licensed under MIT + */ + +import React from 'react'; +import { Component } from 'react'; + +class App extends Component { + render() { + return
Hello
; + } +} + +export default App; +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.js', test_string) + + self.assertEqual(line_offset, 8, "Should skip multiline comment, blank line and imports") + + def test_go_import_block(self): + """Test Go file with import block""" + test_content = b"""// Copyright 2024 +// Licensed under MIT + +package main + +import ( + "fmt" + "os" + _ "github.com/lib/pq" +) + +func main() { + fmt.Println("Hello") +} +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.go', test_string) + + self.assertEqual(line_offset, 11, "Should skip license, package, import block and blank line") + + def test_cpp_include_and_header_guards(self): + """Test C++ file with includes and header guards""" + test_content = b"""/* + * Copyright (c) 2024 + * Licensed under MIT License + */ + +#ifndef MY_HEADER_H +#define MY_HEADER_H + +#include +#include + +class MyClass { +public: + void doSomething(); +}; + +#endif +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.cpp', test_string) + + self.assertGreater(line_offset, 0, "Should skip some header lines") + + def test_java_package_and_imports(self): + """Test Java file with package and imports""" + test_content = b"""/** + * Copyright 2024 + * Licensed under Apache License 2.0 + */ + +package com.example.myapp; + +import java.util.List; +import java.util.ArrayList; +import javax.annotation.Nullable; + +public class MyClass { + private List items; + + public MyClass() { + items = new ArrayList<>(); + } +} +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.java', test_string) + + self.assertGreater(line_offset, 0, "Should skip license, package and imports") + + def test_typescript_with_type_imports(self): + """Test TypeScript file with type imports""" + test_content = b"""// Copyright 2024 +// MIT License + +import type { User } from './types'; +import { Component } from 'react'; +import React from 'react'; + +interface Props { + user: User; +} + +class UserComponent extends Component { + render() { + return
{this.props.user.name}
; + } +} +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.ts', test_string) + + self.assertGreater(line_offset, 0, "Should skip license and imports") + + def test_rust_use_statements(self): + """Test Rust file with use statements""" + test_content = b"""// Copyright 2024 +// Licensed under MIT + +use std::io; +use std::fs::File; +extern crate serde; + +fn main() { + println!("Hello, world!"); +} + +fn another_function() { + // Implementation +} +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.rs', test_string) + + self.assertGreater(line_offset, 0, "Should skip license and use statements") + + def test_python_with_shebang(self): + """Test Python file with shebang""" + test_content = b"""#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2024 + +import sys + +def main(): + pass +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.py', test_string) + + self.assertGreater(line_offset, 0, "Should skip shebang, encoding, license and imports") + + def test_unsupported_language_no_filtering(self): + """Test that unsupported file extensions return original content""" + test_content = b"""Some random content +in an unknown format +that should not be filtered +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.unknown', test_string) + + self.assertEqual(line_offset, 0, "Unsupported files should not be filtered") + + def test_file_with_only_license_and_comments(self): + """Test file that contains only license and comments (no implementation)""" + test_content = b"""# Copyright 2024 +# MIT License +# +# This is just a license file +# with no actual code +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.py', test_string) + + self.assertEqual(line_offset, 0, "Line offset should be 0 when no implementation found") + + def test_max_lines_limit(self): + """Test that max_lines parameter limits output""" + test_content = b"""# Copyright 2024 +# Licensed under MIT +# All rights reserved + +import os +import sys +from pathlib import Path + +# More imports to push implementation beyond line 5 +import json +import asyncio + +def func1(): + pass + +def func2(): + pass +""" + line_filter_limited = HeaderFilter(skip_limit=5, debug=False, quiet=True) + test_string = test_content.decode('utf-8', 'ignore') + line_offset = line_filter_limited.filter('test.py', test_string) + + # Without max_lines, this would be around line 12 (after all imports) + # With max_lines=5, it should be capped at 5 + self.assertEqual(line_offset, 5, "Should cap line_offset at max_lines when implementation starts beyond limit") + + def test_php_namespace_and_use(self): + """Test PHP file with namespace and use statements""" + test_content = b"""/** + * Copyright 2024 + * MIT License + */ + +namespace App\\Controllers; + +use App\\Models\\User; +use Illuminate\\Http\\Request; + +class UserController { + public function index() { + return User::all(); + } +} +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.php', test_string) + + self.assertGreater(line_offset, 0, "Should skip license, namespace and use statements") + + def test_ruby_require_statements(self): + """Test Ruby file with require statements""" + test_content = b"""# Copyright 2024 +# MIT License + +require 'json' +require_relative 'helper' + +class MyClass + def initialize + @data = [] + end +end +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.rb', test_string) + + self.assertGreater(line_offset, 0, "Should skip license and require statements") + + def test_scala_package_and_imports(self): + """Test Scala file with package and imports""" + test_content = b"""/* + * Copyright 2024 + * Apache License 2.0 + */ + +package com.example + +import scala.collection.mutable.ArrayBuffer +import java.util.Date + +object Main { + def main(args: Array[String]): Unit = { + println("Hello") + } +} +""" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.scala', test_string) + + self.assertGreater(line_offset, 0, "Should skip license, package and imports") + + def test_detect_language(self): + """Test language detection from file extensions""" + self.assertEqual(self.line_filter.detect_language('test.py'), 'python') + self.assertEqual(self.line_filter.detect_language('test.js'), 'javascript') + self.assertEqual(self.line_filter.detect_language('test.ts'), 'typescript') + self.assertEqual(self.line_filter.detect_language('test.go'), 'go') + self.assertEqual(self.line_filter.detect_language('test.rs'), 'rust') + self.assertEqual(self.line_filter.detect_language('test.java'), 'java') + self.assertEqual(self.line_filter.detect_language('test.cpp'), 'cpp') + self.assertEqual(self.line_filter.detect_language('test.c'), 'cpp') + self.assertEqual(self.line_filter.detect_language('test.rb'), 'ruby') + self.assertEqual(self.line_filter.detect_language('test.php'), 'php') + self.assertEqual(self.line_filter.detect_language('test.unknown'), None) + + def test_empty_file(self): + """Test handling of empty file""" + test_content = b"" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.py', test_string) + + self.assertEqual(line_offset, 0, "Empty file should have 0 offset") + + def test_utf8_decode_error_handling(self): + """Test handling of files that cannot be decoded as UTF-8""" + # Create content with invalid UTF-8 sequences + test_content = b"\xff\xfe" + b"some content" + test_string = test_content.decode('utf-8', 'ignore') + line_offset = self.line_filter.filter('test.py', test_string) + + # Should return 0 offset when UTF-8 decode fails + self.assertEqual(line_offset, 0, "Should return 0 offset on decode error") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_winnowing.py b/tests/test_winnowing.py index cc26947d..a5b42486 100644 --- a/tests/test_winnowing.py +++ b/tests/test_winnowing.py @@ -388,6 +388,148 @@ def test_file_with_null_bytes_and_newlines(self): # Should generate fh2 (has line endings) self.assertIn('fh2=', wfp) + def test_skip_headers_flag(self): + """Test skip_headers flag functionality.""" + # Sample Python file with headers, imports, and implementation + test_content = b"""# Copyright 2024 SCANOSS +# Licensed under MIT License +# All rights reserved + +import os +import sys +import json +from pathlib import Path + +def function1(): + data = {"key": "value"} + return json.dumps(data) + +def function2(): + path = Path("/tmp") + return str(path) + +class MyClass: + def __init__(self): + self.data = [] + + def add_item(self, item): + self.data.append(item) +""" + + # Test WITHOUT skip_headers + winnowing_no_skip = Winnowing(debug=False, skip_headers=False) + wfp_no_skip = winnowing_no_skip.wfp_for_contents('test.py', False, test_content) + + # Test WITH skip_headers + winnowing_skip = Winnowing(debug=False, skip_headers=True) + wfp_skip = winnowing_skip.wfp_for_contents('test.py', False, test_content) + + print(f'WFP without skip_headers:\n{wfp_no_skip}') + print(f'\nWFP with skip_headers:\n{wfp_skip}') + + # Both should have file= line + self.assertIn('file=', wfp_no_skip) + self.assertIn('file=', wfp_skip) + + # Extract snippet line numbers from both WFPs + def extract_line_numbers(wfp): + lines = wfp.split('\n') + line_numbers = [] + for line in lines: + if '=' in line and line.split('=')[0].isdigit(): + line_numbers.append(int(line.split('=')[0])) + return line_numbers + + lines_no_skip = extract_line_numbers(wfp_no_skip) + lines_skip = extract_line_numbers(wfp_skip) + + # Both should have snippet lines + self.assertGreater(len(lines_no_skip), 0, "Should have snippets without skip_headers") + self.assertGreater(len(lines_skip), 0, "Should have snippets with skip_headers") + + # First line number with skip_headers should be HIGHER (skipped headers/imports) + # Line 10 in the content is "def function1():" which is where real code starts + min_line_no_skip = min(lines_no_skip) + min_line_skip = min(lines_skip) + + print(f'First snippet line without skip_headers: {min_line_no_skip}') + print(f'First snippet line with skip_headers: {min_line_skip}') + + # With skip_headers, first line should be after imports (around line 10+) + # Without skip_headers, first line should be earlier (around line 5-8) + self.assertGreater( + min_line_skip, + min_line_no_skip, + "skip_headers should result in higher starting line number" + ) + + # Verify line 10+ (implementation) appears in skip_headers output + self.assertGreaterEqual( + min_line_skip, + 10, + "With skip_headers, snippets should start at implementation (line 10+)" + ) + + # Verify start_line tag is present in skip_headers output + self.assertIn('start_line=', wfp_skip, "start_line tag should be present with skip_headers") + self.assertNotIn('start_line=', wfp_no_skip, "start_line tag should NOT be present without skip_headers") + + # Extract and validate start_line value + start_line_value = None + for line in wfp_skip.split('\n'): + if line.startswith('start_line='): + start_line_value = int(line.split('=')[1]) + break + + self.assertIsNotNone(start_line_value, "start_line value should be found") + self.assertGreater(start_line_value, 0, "start_line should indicate skipped lines") + print(f'start_line tag value: {start_line_value}') + + def test_skip_headers_with_different_languages(self): + """Test skip_headers with different programming languages.""" + + # JavaScript test + js_content = b"""/* + * Copyright 2024 + * Licensed under MIT + */ + +import React from 'react'; +import { Component } from 'react'; + +class App extends Component { + render() { + return
Hello
; + } +} +""" + winnowing_js = Winnowing(debug=False, skip_headers=True) + wfp_js = winnowing_js.wfp_for_contents('test.js', False, js_content) + + print(f'JavaScript WFP with skip_headers:\n{wfp_js}') + + # Should have snippets starting from class definition (not imports) + self.assertIn('file=', wfp_js) + + # Go test + go_content = b"""// Copyright 2024 +// Licensed under MIT + +package main + +import ( + "fmt" + "os" +) + +func main() { + fmt.Println("Hello") +} +""" + winnowing_go = Winnowing(debug=False, skip_headers=True) + wfp_go = winnowing_go.wfp_for_contents('test.go', False, go_content) + + print(f'Go WFP with skip_headers:\n{wfp_go}') -if __name__ == '__main__': - unittest.main() + # Should have snippets starting from func main (not package/imports) + self.assertIn('file=', wfp_go) \ No newline at end of file diff --git a/tools/linter.sh b/tools/linter.sh index 04d1e76d..bb8a8413 100755 --- a/tools/linter.sh +++ b/tools/linter.sh @@ -2,14 +2,13 @@ # SPDX-License-Identifier: MIT # # Lint Python files changed since merge base with origin/main -# Usage: linter.sh [--fix] [--docker] - +# Usage: linter.sh [--fix] [--docker] [--all] set -e - +RUFF_IMAGE="ghcr.io/astral-sh/ruff:0.14.2" # Parse arguments FIX_FLAG="" USE_DOCKER=false - +ALL_FILES=false while [[ $# -gt 0 ]]; do case $1 in --fix) @@ -20,31 +19,48 @@ while [[ $# -gt 0 ]]; do USE_DOCKER=true shift ;; + --all) + ALL_FILES=true + shift + ;; *) echo "Unknown option: $1" - echo "Usage: $0 [--fix] [--docker]" + echo "Usage: $0 [--fix] [--docker] [--all]" exit 1 ;; esac done - -# Find merge base with origin/main -merge_base=$(git merge-base origin/main HEAD) - -# Get all changed Python files since merge base -files=$(git diff --name-only "$merge_base" HEAD | grep '\.py$' || true) +# Get a list of files to analyse +files="" +if [ "$ALL_FILES" = "true" ] ; then + echo "Analysing all python files..." + files=$(find . -type f -name "*.py" -print) +else + # Find merge base with origin/main + if ! git rev-parse --verify origin/main >/dev/null 2>&1; then + echo "Error: origin/main branch not found. Ensure you have fetched from origin." + exit 1 + fi + merge_base=$(git merge-base origin/main HEAD) + # Get all changed Python files since merge base + files=$(git diff --name-only "$merge_base" HEAD | grep '\.py$' || true) +fi +# Filter out files that match exclude patterns from pyproject.toml +# this is a temporary workaround until we fix all the lint errors +filtered_files=$(echo "$files" | grep -v -E 'tests/|test_.*\.py|src/protoc_gen_swagger/|src/scanoss/api/' || true) # Check if there are any Python files changed -if [ -z "$files" ]; then +if [ -z "$filtered_files" ]; then echo "No Python files changed" exit 0 fi - +file_count=$(echo "${filtered_files}" | wc -l | tr -d ' ') +echo "Analysing ${file_count} files..." # Run linter if [ "$USE_DOCKER" = true ]; then # Run with Docker - docker run --rm -v "$(pwd)":/src -w /src ghcr.io/astral-sh/ruff:0.14.2 check ${files} ${FIX_FLAG} + echo "$filtered_files" | xargs -r docker run --rm -v "$(pwd)":/src -w /src ${RUFF_IMAGE} check ${FIX_FLAG} else # Run locally - python3 -m ruff check ${files} ${FIX_FLAG} + echo "$filtered_files" | xargs -r python3 -m ruff check ${FIX_FLAG} fi \ No newline at end of file From 02636eae15641e17ed7ea7762ae8ef985c5f1f25 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 2 Jan 2026 11:56:16 +0100 Subject: [PATCH 409/489] refactor(scanner): remove dead code scan_wfp_file() never called --- src/scanoss/scanner.py | 105 ----------------------------------------- 1 file changed, 105 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index cec80326..5b131c12 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -767,111 +767,6 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: success = False return success - def scan_wfp_file(self, file: str = None) -> bool: # noqa: PLR0912, PLR0915 - """ - Scan the contents of the specified WFP file (in the current process) - :param file: Scan the contents of the specified WFP file (in the current process) - :return: True if successful, False otherwise - """ - success = True - wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default - if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): - raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}') - file_count = Scanner.__count_files_in_wfp_file(wfp_file) - cur_files = 0 - cur_size = 0 - batch_files = 0 - wfp = '' - max_component = {'name': '', 'hits': 0} - components = {} - self.print_debug(f'Found {file_count} files to process.') - raw_output = '{\n' - file_print = '' - bar_ctx = Bar('Scanning', max=file_count) if (not self.quiet and self.isatty) else nullcontext() - - with bar_ctx as bar: - if bar: - bar.next(0) - with open(wfp_file) as f: - for line in f: - if line.startswith(WFP_FILE_START): - if file_print: - wfp += file_print # Store the WFP for the current file - cur_size = len(wfp.encode('utf-8')) - file_print = line # Start storing the next file - cur_files += 1 - batch_files += 1 - else: - file_print += line # Store the rest of the WFP for this file - l_size = cur_size + len(file_print.encode('utf-8')) - # Hit the max post size, so sending the current batch and continue processing - if l_size >= self.max_post_size and wfp: - self.print_debug( - f'Sending {batch_files} ({cur_files}) of' - f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' - ) - if self.debug and cur_size > self.max_post_size: - Scanner.print_stderr( - f'Warning: Post size {cur_size} greater than limit {self.max_post_size}' - ) - scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store - if bar: - bar.next(batch_files) - if scan_resp is not None: - for key, value in scan_resp.items(): - raw_output += ' "%s":%s,' % (key, json.dumps(value, indent=2)) - for v in value: - if hasattr(v, 'get'): - if v.get('id') != 'none': - vcv = '%s:%s:%s' % (v.get('vendor'), v.get('component'), v.get('version')) - components[vcv] = components[vcv] + 1 if vcv in components else 1 - if max_component['hits'] < components[vcv]: - max_component['name'] = v.get('component') - max_component['hits'] = components[vcv] - else: - Scanner.print_stderr(f'Warning: Unknown value: {v}') - else: - success = False - batch_files = 0 - wfp = '' - if file_print: - wfp += file_print # Store the WFP for the current file - if wfp: - self.print_debug( - f'Sending {batch_files} ({cur_files}) of' - f' {file_count} ({len(wfp.encode("utf-8"))} bytes) files to the ScanOSS API.' - ) - scan_resp = self.scanoss_api.scan(wfp, max_component['name']) # Scan current WFP and store - if bar: - bar.next(batch_files) - first = True - if scan_resp is not None: - for key, value in scan_resp.items(): - if first: - raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2)) - first = False - else: - raw_output += ',\n "%s":%s' % (key, json.dumps(value, indent=2)) - else: - success = False - raw_output += '\n}' - if self.output_format == 'plain': - self.__log_result(raw_output) - elif self.output_format == 'cyclonedx': - cdx = CycloneDx(self.debug, self.scan_output) - cdx.produce_from_str(raw_output) - elif self.output_format == 'spdxlite': - spdxlite = SpdxLite(self.debug, self.scan_output) - success = spdxlite.produce_from_str(raw_output) - elif self.output_format == 'csv': - csvo = CsvOutput(self.debug, self.scan_output) - csvo.produce_from_str(raw_output) - else: - self.print_stderr(f'ERROR: Unknown output format: {self.output_format}') - success = False - - return success - def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) -> bool: """ Scan the given WFP file for whatever scaning options that have been configured From 21a7efea421bd46bef371a4254b066484642bb60 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 2 Jan 2026 12:04:48 +0100 Subject: [PATCH 410/489] feat(scanner)!: remove --no-wfp-output flag and auto WFP generation during scan --- src/scanoss/cli.py | 5 ----- src/scanoss/scanner.py | 41 +++++++++-------------------------------- 2 files changed, 9 insertions(+), 37 deletions(-) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index d38b2517..dc740173 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -170,7 +170,6 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 default=DEFAULT_RETRY, help='Retry limit for API communication (optional - default 5)', ) - p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation') p_scan.add_argument('--dependencies', '-D', action='store_true', help='Add Dependency scanning') p_scan.add_argument('--dependencies-only', action='store_true', help='Run Dependency scanning only') p_scan.add_argument( @@ -1552,9 +1551,6 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 if args.retry < 0: print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.') - if not os.access(os.getcwd(), os.W_OK): # Make sure the current directory is writable. If not disable saving WFP - print_stderr(f'Warning: Current directory is not writable: {os.getcwd()}') - args.no_wfp_output = True if args.ca_cert and not os.path.exists(args.ca_cert): print_stderr(f'Error: Certificate file does not exist: {args.ca_cert}.') sys.exit(1) @@ -1573,7 +1569,6 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 nb_threads=args.threads, post_size=args.post_size, timeout=args.timeout, - no_wfp_file=args.no_wfp_output, all_extensions=args.all_extensions, all_folders=args.all_folders, hidden_files_folders=args.all_hidden, diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 5b131c12..bbdd3f29 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -31,7 +31,6 @@ from typing import Any, Dict, List, Optional import importlib_resources -from progress.bar import Bar from progress.spinner import Spinner from pypac.parser import PACFile @@ -72,7 +71,6 @@ class Scanner(ScanossBase): def __init__( # noqa: PLR0913, PLR0915 self, - wfp: str = None, scan_output: str = None, output_format: str = 'plain', debug: bool = False, @@ -84,7 +82,6 @@ def __init__( # noqa: PLR0913, PLR0915 nb_threads: int = 5, post_size: int = 32, timeout: int = 180, - no_wfp_file: bool = False, all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False, @@ -120,10 +117,8 @@ def __init__( # noqa: PLR0913, PLR0915 skip_folders = [] if skip_extensions is None: skip_extensions = [] - self.wfp = wfp if wfp else 'scanner_output.wfp' self.scan_output = scan_output self.output_format = output_format - self.no_wfp_file = no_wfp_file self.isatty = sys.stderr.isatty() self.all_extensions = all_extensions self.all_folders = all_folders @@ -372,8 +367,6 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext() with spinner_ctx as spinner: - save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan - wfp_list = [] scan_block = '' scan_size = 0 queue_size = 0 @@ -394,8 +387,6 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 if wfp is None or wfp == '': self.print_debug(f'No WFP returned for {to_scan_file}. Skipping.') continue - if save_wfps_for_print: - wfp_list.append(wfp) file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) @@ -429,12 +420,6 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted if file_count > 0: - if save_wfps_for_print: # Write a WFP file if no threading is requested - self.print_debug(f'Writing fingerprints to {self.wfp}') - with open(self.wfp, 'w') as f: - f.write(''.join(wfp_list)) - else: - self.print_debug(f'Skipping writing WFP file {self.wfp}') if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: @@ -642,8 +627,6 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 spinner_ctx = Spinner('Fingerprinting ') if (not self.quiet and self.isatty) else nullcontext() with spinner_ctx as spinner: - save_wfps_for_print = not self.no_wfp_file or not self.threaded_scan - wfp_list = [] scan_block = '' scan_size = 0 queue_size = 0 @@ -663,8 +646,6 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 if wfp is None or wfp == '': self.print_debug(f'No WFP returned for {file}. Skipping.') continue - if save_wfps_for_print: - wfp_list.append(wfp) file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) @@ -699,12 +680,6 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted if file_count > 0: - if save_wfps_for_print: # Write a WFP file if no threading is requested - self.print_debug(f'Writing fingerprints to {self.wfp}') - with open(self.wfp, 'w') as f: - f.write(''.join(wfp_list)) - else: - self.print_debug(f'Skipping writing WFP file {self.wfp}') if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: @@ -767,21 +742,22 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: success = False return success - def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) -> bool: + def scan_wfp_with_options(self, wfp_file: str, deps_file: str, file_map: dict = None) -> bool: """ Scan the given WFP file for whatever scaning options that have been configured - :param wfp: WFP file to scan + :param wfp_file: WFP file to scan :param deps_file: pre-parsed dependency file to decorate :param file_map: mapping of obfuscated files back into originals :return: True if successful, False otherwise """ success = True - wfp_file = wfp if wfp else self.wfp # If a WFP file is specified, use it, otherwise us the default + if not wfp_file: + raise Exception('ERROR: Please specify a WFP file to scan') if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}') if not self.is_file_or_snippet_scan() and not self.is_dependency_scan(): - raise Exception(f'ERROR: No scan options defined to scan WFP: {wfp}') + raise Exception(f'ERROR: No scan options defined to scan WFP: {wfp_file}') if self.scan_output: self.print_msg(f'Writing results to {self.scan_output}...') @@ -796,14 +772,15 @@ def scan_wfp_with_options(self, wfp: str, deps_file: str, file_map: dict = None) success = False return success - def scan_wfp_file_threaded(self, file: str = None) -> bool: + def scan_wfp_file_threaded(self, wfp_file: str) -> bool: """ Scan the contents of the specified WFP file (threaded) - :param file: WFP file to scan (optional) + :param wfp_file: WFP file to scan return: True if successful, False otherwise """ success = True - wfp_file = file if file else self.wfp # If a WFP file is specified, use it, otherwise us the default + if not wfp_file: + raise Exception('ERROR: Please specify a WFP file to scan') if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}') cur_size = 0 From f18d4b2d621521be68752211a84485bae9d37975 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 2 Jan 2026 12:05:04 +0100 Subject: [PATCH 411/489] docs(cli): remove --no-wfp-output flag from documentation --- docs/source/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index dd774c4d..c759e003 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -90,8 +90,6 @@ Scans a directory or file (source code or ``.wfp`` fingerprint file) and shows r - Number of kilobytes to limit the post to while scanning (optional - default 64) * - --timeout , -M - Timeout (in seconds) for API communication (optional - default 120) - * - --no-wfp-output - - Skip WFP file generation * - --all folders - Scan all folders * - --all-extensions From 6d5d4549d42020110d6e4b598ea0b50705b1a750 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 2 Jan 2026 12:26:30 +0100 Subject: [PATCH 412/489] chore(release): prepare for the 1.43.0 release --- CHANGELOG.md | 8 +++++++- src/scanoss/__init__.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a89cf72..04bbd9d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... +## [1.43.0] - 2026-01-02 +### Changed + - Scan command no longer generates `scanner_output.wfp` file + - Removed `--no-wfp-output` flag (no longer needed) + ## [1.42.0] - 2025-12-17 ### Added - Added support for filtering uninteresting data from the beginning of source files. @@ -761,4 +766,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.40.1]: https://github.com/scanoss/scanoss.py/compare/v1.40.0...v1.40.1 [1.41.0]: https://github.com/scanoss/scanoss.py/compare/v1.40.1...v1.41.0 [1.41.1]: https://github.com/scanoss/scanoss.py/compare/v1.41.0...v1.41.1 -[1.42.0]: https://github.com/scanoss/scanoss.py/compare/v1.41.1...v1.42.0 \ No newline at end of file +[1.42.0]: https://github.com/scanoss/scanoss.py/compare/v1.41.1...v1.42.0 +[1.43.0]: https://github.com/scanoss/scanoss.py/compare/v1.42.0...v1.43.0 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 224053eb..eee224f7 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.42.0' +__version__ = '1.43.0' From f953395255da2cbb12957a79c8ed7f5e32488c28 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 2 Jan 2026 12:44:37 +0100 Subject: [PATCH 413/489] fix(lint): suppress "too many branches" in `scan_wfp_file_threaded` Added # noqa: PLR0912 to bypass the linter's branch limit (13 > 12). The extra branches come from separate validation checks that provide clearer error messages to users. Combining them would reduce clarity. --- src/scanoss/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index bbdd3f29..48878080 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -772,7 +772,7 @@ def scan_wfp_with_options(self, wfp_file: str, deps_file: str, file_map: dict = success = False return success - def scan_wfp_file_threaded(self, wfp_file: str) -> bool: + def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 """ Scan the contents of the specified WFP file (threaded) :param wfp_file: WFP file to scan From 17926b204cd22034a934330ea39ae1c62c5f132b Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 5 Jan 2026 12:14:12 +0100 Subject: [PATCH 414/489] chore(cli): deprecate --no-wfp-output flag for backwards compatibility --- CHANGELOG.md | 11 ++++++++--- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 6 ++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04bbd9d1..b1cb4d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... +## [1.43.1] - 2026-01-05 +### Changed +- Restored `--no-wfp-output` flag for backwards compatibility (deprecated, no effect) + ## [1.43.0] - 2026-01-02 ### Changed - - Scan command no longer generates `scanner_output.wfp` file - - Removed `--no-wfp-output` flag (no longer needed) +- Scan command no longer generates `scanner_output.wfp` file +- Removed `--no-wfp-output` flag (no longer needed) ## [1.42.0] - 2025-12-17 ### Added @@ -767,4 +771,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.41.0]: https://github.com/scanoss/scanoss.py/compare/v1.40.1...v1.41.0 [1.41.1]: https://github.com/scanoss/scanoss.py/compare/v1.41.0...v1.41.1 [1.42.0]: https://github.com/scanoss/scanoss.py/compare/v1.41.1...v1.42.0 -[1.43.0]: https://github.com/scanoss/scanoss.py/compare/v1.42.0...v1.43.0 \ No newline at end of file +[1.43.0]: https://github.com/scanoss/scanoss.py/compare/v1.42.0...v1.43.0 +[1.43.1]: https://github.com/scanoss/scanoss.py/compare/v1.43.0...v1.43.1 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index eee224f7..d090807f 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.43.0' +__version__ = '1.43.1' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index dc740173..520a6255 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -186,6 +186,10 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) p_scan.add_argument('--dep-scope-inc', '-dsi', type=str, help='Include dependencies with declared scopes') p_scan.add_argument('--dep-scope-exc', '-dse', type=str, help='Exclude dependencies with declared scopes') + p_scan.add_argument( + '--no-wfp-output', action='store_true', + help='DEPRECATED: Scans no longer generate scanner_output.wfp. Use "fingerprint -o" to create WFP files.' + ) # Sub-command: fingerprint p_wfp = subparsers.add_parser( @@ -1470,6 +1474,8 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 ) parser.parse_args([args.subparser, '-h']) sys.exit(1) + if args.no_wfp_output: + print_stderr('Warning: --no-wfp-output is deprecated and has no effect. It will be removed in a future version') if args.pac and args.proxy: print_stderr('Please specify one of --proxy or --pac, not both') parser.parse_args([args.subparser, '-h']) From e0d3269c1a1061d5ca3238cd976e3543ba9be51b Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 15 Jan 2026 17:10:20 +0100 Subject: [PATCH 415/489] feat(examples): add SDK example for WFP fingerprint generation and scanning --- .gitignore | 1 + examples/sample_code/hello.c | 9 +++++ examples/sample_code/utils.py | 25 +++++++++++++ examples/wfp_scan_example.py | 68 +++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 examples/sample_code/hello.c create mode 100644 examples/sample_code/utils.py create mode 100644 examples/wfp_scan_example.py diff --git a/.gitignore b/.gitignore index fe84723a..2ddf8047 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ docs/build !scanoss-settings-schema.json .DS_Store !scanoss.json +examples/output/ diff --git a/examples/sample_code/hello.c b/examples/sample_code/hello.c new file mode 100644 index 00000000..f5023c4e --- /dev/null +++ b/examples/sample_code/hello.c @@ -0,0 +1,9 @@ +/* + * Sample C file for SCANOSS fingerprinting demo + */ +#include + +int main() { + printf("Hello, World!\n"); + return 0; +} diff --git a/examples/sample_code/utils.py b/examples/sample_code/utils.py new file mode 100644 index 00000000..10ebac36 --- /dev/null +++ b/examples/sample_code/utils.py @@ -0,0 +1,25 @@ +""" +Sample Python file for SCANOSS fingerprinting demo +""" + + +def add(a, b): + """Add two numbers.""" + return a + b + + +def subtract(a, b): + """Subtract two numbers.""" + return a - b + + +def multiply(a, b): + """Multiply two numbers.""" + return a * b + + +def divide(a, b): + """Divide two numbers.""" + if b == 0: + raise ValueError("Cannot divide by zero") + return a / b diff --git a/examples/wfp_scan_example.py b/examples/wfp_scan_example.py new file mode 100644 index 00000000..f830d833 --- /dev/null +++ b/examples/wfp_scan_example.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +""" +SCANOSS SDK Example: WFP Generation and Scanning + +This example demonstrates how to: +1. Generate WFP fingerprints from a folder +2. Save fingerprints to disk +3. Reuse saved fingerprints for multiple scans + +Usage: + python wfp_scan_example.py +""" + +import os + +from scanoss.scanner import Scanner +from scanoss.scantype import ScanType + +# Get the directory where this script is located +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +# Sample paths relative to this script +SAMPLE_CODE_DIR = os.path.join(SCRIPT_DIR, 'sample_code') +OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'output') +WFP_FILE = os.path.join(OUTPUT_DIR, 'fingerprints.wfp') +RESULTS_FILE = os.path.join(OUTPUT_DIR, 'results.json') + + +def main(): + # Create output directory if it doesn't exist + os.makedirs(OUTPUT_DIR, exist_ok=True) + + # Step 1: Create Scanner instance with options + scanner = Scanner( + debug=True, + quiet=False, + scan_output=RESULTS_FILE, # Where to save scan results + # api_key='your-api-key', # Optional: your SCANOSS API key + scan_options=ScanType.SCAN_FILES.value | ScanType.SCAN_SNIPPETS.value, # File and snippet scanning only + ) + + # Step 2: Generate and save WFP fingerprints to disk + print(f'Generating fingerprints from: {SAMPLE_CODE_DIR}') + print(f'Saving fingerprints to: {WFP_FILE}') + scanner.wfp_folder( + scan_dir=SAMPLE_CODE_DIR, + wfp_file=WFP_FILE, + ) + print('Fingerprints generated successfully!\n') + + # Step 3: Reuse the saved WFP for multiple scans + print(f'Scanning using fingerprints: {WFP_FILE}') + print(f'Results will be saved to: {RESULTS_FILE}') + scanner.scan_wfp_with_options( + wfp_file=WFP_FILE, + deps_file='', # No dependency file needed since we disabled dependency scanning + ) + print('Scan completed!\n') + + #You can run additional scans with the same fingerprints + # scanner.scan_wfp_with_options( + # wfp_file=WFP_FILE, + # deps_file='', # No dependency file needed since we disabled dependency scanning + # ) + + +if __name__ == '__main__': + main() From 0f86cb3b9f44769c02dff0dcb59ab44d3e2d8fb4 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 16 Jan 2026 18:55:17 +0100 Subject: [PATCH 416/489] add --wfp-output option to save fingerprints during scan --- src/scanoss/cli.py | 5 +++++ src/scanoss/scanner.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 520a6255..e3b4eaa4 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -190,6 +190,10 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--no-wfp-output', action='store_true', help='DEPRECATED: Scans no longer generate scanner_output.wfp. Use "fingerprint -o" to create WFP files.' ) + p_scan.add_argument( + '--wfp-output', type=str, metavar='FILE', + help='Save fingerprints to specified file during scan' + ) # Sub-command: fingerprint p_wfp = subparsers.add_parser( @@ -1601,6 +1605,7 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 use_grpc=args.grpc, skip_headers=args.skip_headers, skip_headers_limit=args.skip_headers_limit, + wfp_output=args.wfp_output, ) if args.wfp: if not scanner.is_file_or_snippet_scan(): diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 48878080..db6469a7 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -108,6 +108,7 @@ def __init__( # noqa: PLR0913, PLR0915 use_grpc: bool = False, skip_headers: bool = False, skip_headers_limit: int = 0, + wfp_output: str = None, ): """ Initialise scanning class, including Winnowing, ScanossApi, ThreadedScanning @@ -119,6 +120,7 @@ def __init__( # noqa: PLR0913, PLR0915 skip_extensions = [] self.scan_output = scan_output self.output_format = output_format + self.wfp_output = wfp_output self.isatty = sys.stderr.isatty() self.all_extensions = all_extensions self.all_folders = all_folders @@ -373,6 +375,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False + wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) for to_scan_file in to_scan_files: @@ -387,6 +390,8 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 if wfp is None or wfp == '': self.print_debug(f'No WFP returned for {to_scan_file}. Skipping.') continue + if wfp_list is not None: + wfp_list.append(wfp) file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) @@ -420,6 +425,10 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted if file_count > 0: + if wfp_list is not None: + self.print_debug(f'Writing fingerprints to {self.wfp_output}') + with open(self.wfp_output, 'w') as f: + f.write(''.join(wfp_list)) if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: @@ -633,6 +642,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False + wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified to_scan_files = file_filters.get_filtered_files_from_files(files) for file in to_scan_files: @@ -646,6 +656,8 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 if wfp is None or wfp == '': self.print_debug(f'No WFP returned for {file}. Skipping.') continue + if wfp_list is not None: + wfp_list.append(wfp) file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) @@ -680,6 +692,10 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted if file_count > 0: + if wfp_list is not None: + self.print_debug(f'Writing fingerprints to {self.wfp_output}') + with open(self.wfp_output, 'w') as f: + f.write(''.join(wfp_list)) if self.threaded_scan: success = self.__run_scan_threaded(scan_started, file_count) else: From 5cd1e290169683fe98a6d255e326d07a4c8bcc45 Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Thu, 22 Jan 2026 13:44:34 +0100 Subject: [PATCH 417/489] feat: use SCANOSS base url instead of /scan/direct endpoint (#179) * feat: use SCANOSS base url instead of /scan/direct endpoint * fix: lint errors * chore: update scanoss.json * chore: remove innecessary comments --- CHANGELOG.md | 13 ++++++++- PACKAGE.md | 4 +-- docs/source/index.rst | 4 +-- scanoss.json | 4 ++- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 2 +- src/scanoss/scanossapi.py | 57 ++++++++++++++++++++++++++++++--------- 7 files changed, 65 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1cb4d8d..cb06195d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... +## [1.44.0] - 2026-01-22 +### Changed +- Refactored `--apiurl` parameter to accept base URLs instead of full endpoint URLs + - Now accepts `https://api.scanoss.com` instead of `https://api.scanoss.com/scan/direct` + - Automatically appends `/scan/direct` endpoint path + - Backward compatible: detects and warns when full endpoint URLs are provided + - Uses `urllib.parse` for robust URL handling (ports, IPv6, encoded characters) + - Updated CLI help text and documentation to reflect base URL format + - Applies to both CLI arguments and `SCANOSS_SCAN_URL` environment variable + ## [1.43.1] - 2026-01-05 ### Changed - Restored `--no-wfp-output` flag for backwards compatibility (deprecated, no effect) @@ -772,4 +782,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.41.1]: https://github.com/scanoss/scanoss.py/compare/v1.41.0...v1.41.1 [1.42.0]: https://github.com/scanoss/scanoss.py/compare/v1.41.1...v1.42.0 [1.43.0]: https://github.com/scanoss/scanoss.py/compare/v1.42.0...v1.43.0 -[1.43.1]: https://github.com/scanoss/scanoss.py/compare/v1.43.0...v1.43.1 \ No newline at end of file +[1.43.1]: https://github.com/scanoss/scanoss.py/compare/v1.43.0...v1.43.1 +[1.44.0]: https://github.com/scanoss/scanoss.py/compare/v1.43.1...v1.44.0 diff --git a/PACKAGE.md b/PACKAGE.md index 620f2984..b2e55309 100644 --- a/PACKAGE.md +++ b/PACKAGE.md @@ -117,8 +117,8 @@ if __name__ == "__main__": ``` ## Scanning URL and API Key -By Default, scanoss uses the API URL endpoint for SCANOSS OSS KB: https://api.osskb.org/scan/direct. -This API does not require an API key. +By Default, scanoss uses the API base URL for SCANOSS OSS KB: https://api.osskb.org. +The `/scan/direct` endpoint is automatically appended. This API does not require an API key. These values can be changed from the command line using: ```bash diff --git a/docs/source/index.rst b/docs/source/index.rst index c759e003..e3cacb16 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -107,7 +107,7 @@ Scans a directory or file (source code or ``.wfp`` fingerprint file) and shows r * - --sc-timeout - Timeout (in seconds) for Scancode to complete (optional - default 600) * - --apiurl - - SCANOSS API URL (optional - default https://api.osskb.org/api/scan/direct) + - SCANOSS API base URL (optional - default https://api.osskb.org) * - --ignore-cert-errors - Ignore certificate errors * - --key , -k @@ -119,7 +119,7 @@ Scans a directory or file (source code or ``.wfp`` fingerprint file) and shows r * - --ca-cert - Alternative certificate PEM file, can also use the environment variables ``REQUEST_CA_BUNDLE`` and ``GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`` (optional) * - --api2url - - SCANOSS gRPC API 2.0 URL (optional - default https://api.osskb.org/api/scan/direct) + - SCANOSS gRPC API 2.0 base URL (optional - default https://api.osskb.org) * - --grpc-proxy - GRPC Proxy URL to use for connections, can also us the environment variable ``GRPC_PROXY`` (optional) diff --git a/scanoss.json b/scanoss.json index 954cd89a..c46a924e 100644 --- a/scanoss.json +++ b/scanoss.json @@ -15,9 +15,11 @@ "include": [ { "purl": "pkg:github/scanoss/scanoss.py" + }, + { + "purl": "pkg:pypi/scanoss" } ], "remove": [] } } - diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index d090807f..b7db689a 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.43.1' +__version__ = '1.44.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index e3b4eaa4..57c280be 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1080,7 +1080,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 # Global Scan command options for p in [p_scan, p_cs]: p.add_argument( - '--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)' + '--apiurl', type=str, help='SCANOSS API base URL (optional - default: https://api.osskb.org)' ) # Global Scan/Fingerprint filter options diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index f077585b..1e194da4 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -29,6 +29,7 @@ import time import uuid from json.decoder import JSONDecodeError +from urllib.parse import urlparse, urlunparse import requests import urllib3 @@ -40,8 +41,9 @@ from .constants import DEFAULT_TIMEOUT, MIN_TIMEOUT from .scanossbase import ScanossBase -DEFAULT_URL = 'https://api.osskb.org/scan/direct' # default free service URL -DEFAULT_URL2 = 'https://api.scanoss.com/scan/direct' # default premium service URL +DEFAULT_URL = 'https://api.osskb.org' # default free service base URL +DEFAULT_URL2 = 'https://api.scanoss.com' # default premium service base URL +SCAN_ENDPOINT = '/scan/direct' # scan endpoint path SCANOSS_SCAN_URL = os.environ.get('SCANOSS_SCAN_URL') if os.environ.get('SCANOSS_SCAN_URL') else DEFAULT_URL SCANOSS_API_KEY = os.environ.get('SCANOSS_API_KEY') if os.environ.get('SCANOSS_API_KEY') else '' @@ -52,6 +54,33 @@ class ScanossApi(ScanossBase): Currently support posting scan requests to the SCANOSS streaming API """ + def normalize_api_url(self, url: str) -> str: + """ + Normalize API URL to ensure it's a base URL with the scan endpoint appended. + + If the URL contains a path component (e.g., /scan/direct), a warning is emitted + and the path is stripped to use only the base URL. + + :param url: Input URL (can be base URL or full endpoint URL) + :return: Normalized URL with /scan/direct endpoint + """ + if not url: + return url + + url = url.strip() + parsed = urlparse(url) + + if parsed.path and parsed.path != '/': + self.print_stderr( + f"Warning: URL '{url}' contains path '{parsed.path}'. " + f"Using base URL only: '{parsed.scheme}://{parsed.netloc}'" + ) + base_url = urlunparse((parsed.scheme, parsed.netloc, '', '', '', '')) + else: + base_url = url.rstrip('/') + + return f'{base_url}{SCAN_ENDPOINT}' + def __init__( # noqa: PLR0912, PLR0913, PLR0915 self, scan_format: str = None, @@ -74,7 +103,7 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 Initialise the SCANOSS API :param scan_format: Scan format (default plain) :param flags: Scanning flags (default None) - :param url: API URL (default https://api.osskb.org/scan/direct) + :param url: API base URL (default https://api.osskb.org). The /scan/direct endpoint is automatically appended. :param api_key: API Key (default None) :param debug: Enable debug (default False) :param trace: Enable trace (default False) @@ -95,11 +124,11 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 self.ignore_cert_errors = ignore_cert_errors self.req_headers = req_headers if req_headers else {} self.headers = {} - # Set the correct URL/API key combination - self.url = url if url else SCANOSS_SCAN_URL + base_url = url if url else SCANOSS_SCAN_URL self.api_key = api_key if api_key else SCANOSS_API_KEY if self.api_key and not url and not os.environ.get('SCANOSS_SCAN_URL'): - self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + base_url = DEFAULT_URL2 + self.url = self.normalize_api_url(base_url) if ver_details: self.headers['x-scanoss-client'] = ver_details if self.api_key: @@ -113,7 +142,7 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 if self.trace: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) http_client.HTTPConnection.debuglevel = 1 - if pac and not proxy: # Setup PAC session if requested (and no proxy has been explicitly set) + if pac and not proxy: self.print_debug('Setting up PAC session...') self.session = PACSession(pac=pac) else: @@ -269,19 +298,21 @@ def set_sbom(self, sbom): def load_generic_headers(self, url): """ - Adds custom headers from req_headers to the headers collection. + Adds custom headers from req_headers to the headers collection. - If x-api-key is present and no URL is configured (directly or via - environment), sets URL to the premium endpoint (DEFAULT_URL2). - """ + If x-api-key is present and no URL is configured (directly or via + environment), sets URL to the premium endpoint (DEFAULT_URL2). + """ if self.req_headers: # Load generic headers for key, value in self.req_headers.items(): - if key == 'x-api-key': # Set premium URL if x-api-key header is set + if key == 'x-api-key': # Set premium URL if x-api-key header is set if not url and not os.environ.get('SCANOSS_SCAN_URL'): - self.url = DEFAULT_URL2 # API key specific and no alternative URL, so use the default premium + # API key specific and no alternative URL, so use the default premium + self.url = self.normalize_api_url(DEFAULT_URL2) self.api_key = value self.headers[key] = value + # # End of ScanossApi Class # From 962bca961e066e471698d2906a72cd86cffe85a6 Mon Sep 17 00:00:00 2001 From: Agustin Groh <77737320+agustingroh@users.noreply.github.com> Date: Mon, 2 Feb 2026 11:19:34 -0300 Subject: [PATCH 418/489] Chore/scan tunning parameters * chore:SP-3846 updated scannos settings schema with scan engine tunning parameters * chore:SP-3846 implement scan tunning parameters * chore: upgrade version to v1.45.0 --- CHANGELOG.md | 12 + CLIENT_HELP.md | 145 ++++++- .../_static/scanoss-settings-schema.json | 97 +++++ scanoss.json | 4 + src/scanoss/cli.py | 59 ++- src/scanoss/data/scanoss-settings-schema.json | 94 +++++ src/scanoss/scan_settings_builder.py | 311 +++++++++++++++ src/scanoss/scanner.py | 77 +++- src/scanoss/scanoss_settings.py | 96 +++++ src/scanoss/scanossapi.py | 58 +++ src/scanoss/scanpostprocessor.py | 24 +- tests/data/scanoss.json | 23 ++ tests/scanossapi-test.py | 2 +- tests/test_scan_settings_builder.py | 362 ++++++++++++++++++ 14 files changed, 1327 insertions(+), 37 deletions(-) create mode 100644 src/scanoss/scan_settings_builder.py create mode 100644 tests/test_scan_settings_builder.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cb06195d..4166b69f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... +## [1.45.0] - 2026-02-02 +### Added +- Added scan engine tuning parameters for snippet matching: + - `--min-snippet-hits` - Minimum snippet hits required (0 defers to server config) + - `--min-snippet-lines` - Minimum snippet lines required (0 defers to server config) + - `--ranking` - Enable/disable result ranking (unset/true/false) + - `--ranking-threshold` - Ranking threshold value (-1 to 10, -1 defers to server config) + - `--honour-file-exts` - Honour file extensions during matching (unset/true/false) +- Added `file_snippet` section to scanoss.json settings schema for configuring tuning parameters +- Added `ScanSettingsBuilder` class for merging CLI and settings file configurations with priority: CLI > file_snippet > root settings + ## [1.44.0] - 2026-01-22 ### Changed - Refactored `--apiurl` parameter to accept base URLs instead of full endpoint URLs @@ -784,3 +795,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.43.0]: https://github.com/scanoss/scanoss.py/compare/v1.42.0...v1.43.0 [1.43.1]: https://github.com/scanoss/scanoss.py/compare/v1.43.0...v1.43.1 [1.44.0]: https://github.com/scanoss/scanoss.py/compare/v1.43.1...v1.44.0 +[1.45.0]: https://github.com/scanoss/scanoss.py/compare/v1.44.0...v1.45.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 877c6b8a..ef4a36b8 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -259,6 +259,150 @@ Multiple Headers: You can specify any number of custom headers by repeating the scanoss-py scan src -hdr "x-api-key:12345" -hdr "Authorization: Bearer " ``` +### Scan with Snippet Tuning Options +The following flags allow you to fine-tune snippet matching behavior during scanning. These options can be set via command line flags or in the `scanoss.json` configuration file under `settings.file_snippet`. + +#### Set minimum snippet hits +Require at least 5 snippet hits for a match. A value of 0 defers to server configuration: +```bash +scanoss-py scan -o scan-results.json --min-snippet-hits 5 src +``` +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "min_snippet_hits": 5 + } + } +} +``` + +#### Set minimum snippet lines +Require at least 3 snippet lines for a match. A value of 0 defers to server configuration: +```bash +scanoss-py scan -o scan-results.json --min-snippet-lines 3 src +``` +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "min_snippet_lines": 3 + } + } +} +``` + +#### Enable or disable ranking +Enable ranking to prioritize results: +```bash +scanoss-py scan -o scan-results.json --ranking true src +``` +Disable ranking: +```bash +scanoss-py scan -o scan-results.json --ranking false src +``` +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "ranking_enabled": true + } + } +} +``` + +#### Set ranking threshold +Set the ranking threshold to 5 (valid range: -1 to 10). A value of -1 defers to server configuration: +```bash +scanoss-py scan -o scan-results.json --ranking-threshold=5 src +``` +Note: Use `=` syntax for negative values: `--ranking-threshold=-1` + +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "ranking_threshold": 5 + } + } +} +``` + +#### Honour file extensions +Control whether file extensions are considered during matching: +```bash +scanoss-py scan -o scan-results.json --honour-file-exts true src +``` +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "honour_file_exts": true + } + } +} +``` + +#### Skip headers +Skip license headers, comments and imports at the beginning of files during snippet scanning: +```bash +scanoss-py scan -o scan-results.json --skip-headers src +``` +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "skip_headers": true + } + } +} +``` + +#### Skip headers limit +Set the maximum number of lines to skip when filtering headers (requires `--skip-headers`): +```bash +scanoss-py scan -o scan-results.json --skip-headers --skip-headers-limit 50 src +``` +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "skip_headers": true, + "skip_headers_limit": 50 + } + } +} +``` + +#### Combine multiple tuning options +You can combine multiple tuning options in a single scan: +```bash +scanoss-py scan -o scan-results.json --min-snippet-hits 5 --min-snippet-lines 3 --ranking true --ranking-threshold=5 src +``` +**scanoss.json configuration:** +```json +{ + "settings": { + "file_snippet": { + "min_snippet_hits": 5, + "min_snippet_lines": 3, + "ranking_enabled": true, + "ranking_threshold": 5, + "honour_file_exts": true, + "skip_headers": true, + "skip_headers_limit": 50 + } + } +} +``` + ### Converting RAW results into other formats The following command provides the capability to convert the RAW scan results from a SCANOSS scan into multiple different formats, including CycloneDX, SPDX Lite, CSV and GitLab Code Quality Report. For the full set of formats, please run: @@ -296,7 +440,6 @@ scanoss-py comp search "jquery" -hdr "x-api-key:12345" scanoss-py comp vulns "jquery@3.6.0" -hdr "x-api-key:12345" -hdr "custom-header:value" scanoss-py comp crypto --purl "pkg:github/madler/pigz" -header "x-api-key:12345" - #### Component Vulnerabilities The following command provides the capability to search the SCANOSS KB for component vulnerabilities: ```bash diff --git a/docs/source/_static/scanoss-settings-schema.json b/docs/source/_static/scanoss-settings-schema.json index e9e2cdc3..7cabb8ef 100644 --- a/docs/source/_static/scanoss-settings-schema.json +++ b/docs/source/_static/scanoss-settings-schema.json @@ -139,6 +139,103 @@ } } } + }, + "proxy": { + "type": "object", + "description": "Proxy configuration for API requests", + "properties": { + "host": { + "type": "string", + "description": "Proxy host URL" + } + } + }, + "http_config": { + "type": "object", + "description": "HTTP configuration for API requests", + "properties": { + "base_uri": { + "type": "string", + "description": "Base URI for API requests" + }, + "ignore_cert_errors": { + "type": "boolean", + "description": "Whether to ignore certificate errors" + } + } + }, + "file_snippet": { + "type": "object", + "description": "File snippet scanning configuration", + "properties": { + "proxy": { + "type": "object", + "description": "Proxy configuration for file snippet requests", + "properties": { + "host": { + "type": "string", + "description": "Proxy host URL" + } + } + }, + "http_config": { + "type": "object", + "description": "HTTP configuration for file snippet requests", + "properties": { + "base_uri": { + "type": "string", + "description": "Base URI for file snippet API requests" + }, + "ignore_cert_errors": { + "type": "boolean", + "description": "Whether to ignore certificate errors" + } + } + }, + "ranking_enabled": { + "type": ["boolean", "null"], + "description": "Enable/disable ranking", + "default": null + }, + "ranking_threshold": { + "type": ["integer", "null"], + "description": "Ranking threshold value. A value of -1 defers to server configuration", + "minimum": -1, + "maximum": 10, + "default": -1 + }, + "min_snippet_hits": { + "type": "integer", + "description": "Minimum snippet hits required", + "minimum": 0, + "default": 0 + }, + "min_snippet_lines": { + "type": "integer", + "description": "Minimum snippet lines required", + "minimum": 0, + "default": 0 + }, + "honour_file_exts": { + "type": ["boolean", "null"], + "description": "Ignores file extensions. When not set, defers to server configuration.", + "default": true + }, + "dependency_analysis": { + "type": "boolean", + "description": "Enable dependency analysis" + }, + "skip_headers": { + "type": "boolean", + "description": "Skip license headers, comments and imports at the beginning of files", + "default": false + }, + "skip_headers_limit": { + "type": "integer", + "description": "Maximum number of lines to skip when filtering headers", + "default": 0 + } + } } } }, diff --git a/scanoss.json b/scanoss.json index c46a924e..08f20a82 100644 --- a/scanoss.json +++ b/scanoss.json @@ -18,8 +18,12 @@ }, { "purl": "pkg:pypi/scanoss" + }, + { + "purl": "pkg:github/scanoss/scanoss-winnowing.py" } ], "remove": [] } } + diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 57c280be..93bc5011 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -195,6 +195,40 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Save fingerprints to specified file during scan' ) + # Snippet tuning options + p_scan.add_argument( + '--min-snippet-hits', + type=int, + default=None, + help='Minimum snippet hits required. A value of 0 defers to server configuration (optional)', + ) + p_scan.add_argument( + '--min-snippet-lines', + type=int, + default=None, + help='Minimum snippet lines required. A value of 0 defers to server configuration (optional)', + ) + p_scan.add_argument( + '--ranking', + type=str, + choices=['unset' ,'true', 'false'], + default='unset', + help='Enable or disable ranking (optional - default: server configuration)', + ) + p_scan.add_argument( + '--ranking-threshold', + type=int, + default=-1, + help='Ranking threshold value. Valid range: -1 to 10. A value of -1 defers to server configuration (optional)', + ) + p_scan.add_argument( + '--honour-file-exts', + type=str, + choices=['unset','true', 'false'], + default='unset', + help='Honour file extensions during scanning. When not set, defers to server configuration (optional)', + ) + # Sub-command: fingerprint p_wfp = subparsers.add_parser( 'fingerprint', @@ -1381,11 +1415,11 @@ def wfp(parser, args): initialise_empty_file(args.output) # Load scan settings - scan_settings = None + scanoss_settings = None if not args.skip_settings_file: - scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) + scanoss_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) try: - scan_settings.load_json_file(args.settings, args.scan_dir) + scanoss_settings.load_json_file(args.settings, args.scan_dir) except ScanossSettingsError as e: print_stderr(f'Error: {e}') sys.exit(1) @@ -1407,7 +1441,7 @@ def wfp(parser, args): skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, - scan_settings=scan_settings, + scanoss_settings=scanoss_settings, skip_headers=args.skip_headers, skip_headers_limit=args.skip_headers_limit, ) @@ -1491,20 +1525,20 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 print_stderr('ERROR: Cannot specify both --settings and --skip-file-settings options.') sys.exit(1) # Figure out which settings (if any) to load before processing - scan_settings = None + scanoss_settings = None if not args.skip_settings_file: - scan_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) + scanoss_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) try: if args.identify: - scan_settings.load_json_file(args.identify, args.scan_dir).set_file_type('legacy').set_scan_type( + scanoss_settings.load_json_file(args.identify, args.scan_dir).set_file_type('legacy').set_scan_type( 'identify' ) elif args.ignore: - scan_settings.load_json_file(args.ignore, args.scan_dir).set_file_type('legacy').set_scan_type( + scanoss_settings.load_json_file(args.ignore, args.scan_dir).set_file_type('legacy').set_scan_type( 'blacklist' ) else: - scan_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new') + scanoss_settings.load_json_file(args.settings, args.scan_dir).set_file_type('new') except ScanossSettingsError as e: print_stderr(f'Error: {e}') @@ -1600,9 +1634,14 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 skip_md5_ids=args.skip_md5, strip_hpsm_ids=args.strip_hpsm, strip_snippet_ids=args.strip_snippet, - scan_settings=scan_settings, + scanoss_settings=scanoss_settings, req_headers=process_req_headers(args.header), use_grpc=args.grpc, + min_snippet_hits=args.min_snippet_hits, + min_snippet_lines=args.min_snippet_lines, + ranking=args.ranking, + ranking_threshold=args.ranking_threshold, + honour_file_exts=args.honour_file_exts, skip_headers=args.skip_headers, skip_headers_limit=args.skip_headers_limit, wfp_output=args.wfp_output, diff --git a/src/scanoss/data/scanoss-settings-schema.json b/src/scanoss/data/scanoss-settings-schema.json index e9e2cdc3..124f9c66 100644 --- a/src/scanoss/data/scanoss-settings-schema.json +++ b/src/scanoss/data/scanoss-settings-schema.json @@ -139,6 +139,100 @@ } } } + }, + "file_snippet": { + "type": "object", + "description": "File snippet scanning configuration", + "properties": { + "proxy": { + "type": "object", + "description": "Proxy configuration for file snippet requests", + "properties": { + "host": { + "type": "string", + "description": "Proxy host URL" + } + } + }, + "http_config": { + "type": "object", + "description": "HTTP configuration for file snippet requests", + "properties": { + "base_uri": { + "type": "string", + "description": "Base URI for file snippet API requests" + }, + "ignore_cert_errors": { + "type": "boolean", + "description": "Whether to ignore certificate errors" + } + } + }, + "ranking_enabled": { + "type": ["boolean", "null"], + "description": "Enable/disable ranking", + "default": null + }, + "ranking_threshold": { + "type": ["integer", "null"], + "description": "Ranking threshold value. A value of -1 defers to server configuration", + "minimum": -1, + "maximum": 99, + "default": 0 + }, + "min_snippet_hits": { + "type": "integer", + "description": "Minimum snippet hits required", + "minimum": 0, + "default": 0 + }, + "min_snippet_lines": { + "type": "integer", + "description": "Minimum snippet lines required", + "minimum": 0, + "default": 0 + }, + "honour_file_exts": { + "type": ["boolean", "null"], + "description": "Ignores file extensions. When not set, defers to server configuration.", + "default": true + }, + "dependency_analysis": { + "type": "boolean", + "description": "Enable dependency analysis" + }, + "skip_headers": { + "type": "boolean", + "description": "Skip license headers, comments and imports at the beginning of files", + "default": false + }, + "skip_headers_limit": { + "type": "integer", + "description": "Maximum number of lines to skip when filtering headers", + "default": 0 + } + } + }, + "hpfm": { + "type": "object", + "description": "HPFM (High Precision Folder Matching) configuration", + "properties": { + "ranking_enabled": { + "type": "boolean", + "description": "Enable ranking for HPFM" + }, + "ranking_threshold": { + "type": ["integer", "null"], + "description": "Ranking threshold value. A value of -1 defers to server configuration", + "minimum": -1, + "maximum": 99, + "default": 0 + } + } + }, + "container": { + "type": "object", + "description": "Container scanning configuration" } } }, diff --git a/src/scanoss/scan_settings_builder.py b/src/scanoss/scan_settings_builder.py new file mode 100644 index 00000000..d442ba9d --- /dev/null +++ b/src/scanoss/scan_settings_builder.py @@ -0,0 +1,311 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +from typing import TYPE_CHECKING, Optional + +from .scanossbase import ScanossBase + +if TYPE_CHECKING: + from .scanoss_settings import ScanossSettings + +MAX_RANKING_THRESHOLD = 10 + + +class ScanSettingsBuilder(ScanossBase): + """Builder class for merging CLI arguments with scanoss.json settings file values. + + This class implements an API for merging scan configuration + from multiple sources with the following priority order: + 1. settings.file_snippet section in scanoss.json (highest priority) + 2. settings section in scanoss.json (middle priority) + 3. CLI arguments (lowest priority - used as fallback) + + Attributes: + proxy: Merged proxy host URL + url: Merged API base URL + ignore_cert_errors: Whether to ignore SSL certificate errors + min_snippet_hits: Minimum snippet hits required for matching + min_snippet_lines: Minimum snippet lines required for matching + honour_file_exts: Whether to honour file extensions during scanning + ranking: Whether ranking is enabled + ranking_threshold: Ranking threshold value + """ + + def __init__( + self, + scanoss_settings: 'ScanossSettings | None', + debug: bool = False, + trace: bool = False, + quiet: bool = False, + ): + """Initialize the builder with optional scanoss settings. + + Args: + scanoss_settings: ScanossSettings instance loaded from scanoss.json, + or None if no settings file is available. + debug: Enable debug output + trace: Enable trace output + quiet: Enable quiet mode + """ + super().__init__(debug=debug, trace=trace, quiet=quiet) + self.scanoss_settings = scanoss_settings + # Merged values + self.proxy: Optional[str] = None + self.url: Optional[str] = None + self.ignore_cert_errors: bool = False + self.min_snippet_hits: Optional[int] = None + self.min_snippet_lines: Optional[int] = None + self.honour_file_exts: Optional[any] = None + self.ranking: Optional[any] = None + self.ranking_threshold: Optional[int] = None + + def with_proxy(self, cli_value: str = None) -> 'ScanSettingsBuilder': + """Set proxy host with priority: file_snippet.proxy.host > settings.proxy.host > CLI. + + Args: + cli_value: Proxy host from CLI argument (e.g., 'http://proxy:8080') + + Returns: + Self for method chaining + """ + self.proxy = self._merge_with_priority( + cli_value, + self._get_proxy_host(self._get_file_snippet_proxy()), + self._get_proxy_host(self._get_root_proxy()) + ) + return self + + def with_url(self, cli_value: str = None) -> 'ScanSettingsBuilder': + """Set API base URL with priority: file_snippet.http_config.base_uri > settings.http_config.base_uri > CLI. + + Args: + cli_value: API base URL from CLI argument (e.g., 'https://api.scanoss.com') + + Returns: + Self for method chaining + """ + self.url = self._merge_with_priority( + cli_value, + self._get_file_snippet_http_config_value('base_uri'), + self._get_http_config_value('base_uri') + ) + return self + + def with_ignore_cert_errors(self, cli_value: bool = False) -> 'ScanSettingsBuilder': + """Set ignore_cert_errors with priority: CLI True > file_snippet > settings > False. + + Note: CLI value only takes effect if True (flag present). False means + the flag was not provided, so settings file values are checked. + + Args: + cli_value: Whether to ignore SSL certificate errors from CLI flag + + Returns: + Self for method chaining + """ + result = self._merge_with_priority( + cli_value if cli_value else None, + self._get_file_snippet_http_config_value('ignore_cert_errors'), + self._get_http_config_value('ignore_cert_errors') + ) + self.ignore_cert_errors = result if result is not None else False + return self + + def with_min_snippet_hits(self, cli_value: int = None) -> 'ScanSettingsBuilder': + """Set minimum snippet hits with priority: settings.file_snippet.min_snippet_hits > CLI. + + Minimum allowed value is 0. Values below 0 will be clamped and logged. + + Args: + cli_value: Minimum snippet hits from CLI argument + + Returns: + Self for method chaining + """ + self.min_snippet_hits = self._merge_cli_with_settings( + cli_value, + self._get_file_snippet_setting('min_snippet_hits') + ) + if self.min_snippet_hits is not None and self.min_snippet_hits < 0: + self.print_msg( + f'WARNING: min-snippet-hits value {self.min_snippet_hits} is below minimum allowed (0). ' + f'Setting to 0.' + ) + self.min_snippet_hits = 0 + return self + + def with_min_snippet_lines(self, cli_value: int = None) -> 'ScanSettingsBuilder': + """Set minimum snippet lines with priority: settings.file_snippet.min_snippet_lines > CLI. + + Minimum allowed value is 0. Values below 0 will be clamped and logged. + + Args: + cli_value: Minimum snippet lines from CLI argument + + Returns: + Self for method chaining + """ + self.min_snippet_lines = self._merge_cli_with_settings( + cli_value, + self._get_file_snippet_setting('min_snippet_lines') + ) + if self.min_snippet_lines is not None and self.min_snippet_lines < 0: + self.print_msg( + f'WARNING: min-snippet-lines value {self.min_snippet_lines} is below minimum allowed (0). ' + f'Setting to 0.' + ) + self.min_snippet_lines = 0 + return self + + def with_honour_file_exts(self, cli_value: str = None) -> 'ScanSettingsBuilder': + """Set honour_file_exts with priority: settings.file_snippet.honour_file_exts > CLI. + + Args: + cli_value: String 'true', 'false', or 'unset' from CLI argument + + Returns: + Self for method chaining + """ + self.honour_file_exts = self._merge_cli_with_settings( + cli_value, + self._get_file_snippet_setting('honour_file_exts') + ) + ## Convert to boolean + if self.honour_file_exts is not None and self.honour_file_exts!= 'unset': + self.honour_file_exts = self._str_to_bool(self.honour_file_exts) + return self + + def with_ranking(self, cli_value: str = None) -> 'ScanSettingsBuilder': + """Set ranking enabled with priority: settings.file_snippet.ranking_enabled > CLI. + + Args: + cli_value: String 'true', 'false', or 'unset' from CLI argument + + Returns: + Self for method chaining + """ + self.ranking = self._merge_cli_with_settings( + cli_value, + self._get_file_snippet_setting('ranking_enabled') + ) + if self.ranking is not None and self.ranking != 'unset': + self.ranking = self._str_to_bool(self.ranking) + return self + + def with_ranking_threshold(self, cli_value: int = None) -> 'ScanSettingsBuilder': + """Set ranking threshold with priority: settings.file_snippet.ranking_threshold > CLI. + + Valid range is -1 to 10. Values outside this range will be clamped and logged. + + Args: + cli_value: Ranking threshold from CLI argument + + Returns: + Self for method chaining + """ + self.ranking_threshold = self._merge_cli_with_settings( + cli_value, + self._get_file_snippet_setting('ranking_threshold') + ) + if self.ranking_threshold is not None: + if self.ranking_threshold > MAX_RANKING_THRESHOLD: + self.print_msg( + f'WARNING: ranking-threshold value {self.ranking_threshold} exceeds maximum allowed ' + f'({MAX_RANKING_THRESHOLD}). Setting to {MAX_RANKING_THRESHOLD}.' + ) + self.ranking_threshold = MAX_RANKING_THRESHOLD + elif self.ranking_threshold < -1: + self.print_msg( + f'WARNING: ranking-threshold value {self.ranking_threshold} is below minimum allowed (-1). ' + f'Setting to -1.' + ) + self.ranking_threshold = -1 + return self + + # Private helper methods + @staticmethod + def _merge_with_priority(cli_value, file_snippet_value, root_value): + """Merge with priority: file_snippet > root settings > CLI""" + if file_snippet_value is not None: + return file_snippet_value + if root_value is not None: + return root_value + return cli_value + + @staticmethod + def _merge_cli_with_settings(cli_value, settings_value): + """Merge CLI value with settings, with settings taking priority over CLI. + + Returns settings_value if not None, otherwise falls back to cli_value. + """ + if settings_value is not None: + return settings_value + return cli_value + + + @staticmethod + def _str_to_bool(value: str) -> Optional[bool]: + """Convert string 'true'/'false' to boolean.""" + if value is None: + return None + if isinstance(value, bool): + return value + return value.lower() == 'true' + + # Methods to extract values from scanoss_settings + def _get_file_snippet_setting(self, key: str): + """Get a setting from the file_snippet section.""" + if not self.scanoss_settings: + return None + return self.scanoss_settings.get_file_snippet_settings().get(key) + + def _get_file_snippet_proxy(self): + """Get proxy config from file_snippet section.""" + return self.scanoss_settings.get_file_snippet_proxy() if self.scanoss_settings else None + + def _get_root_proxy(self): + """Get proxy config from root settings section.""" + return self.scanoss_settings.get_proxy() if self.scanoss_settings else None + + @staticmethod + def _get_proxy_host(proxy_config) -> Optional[str]: + """Extract host from proxy configuration dict.""" + if proxy_config is None: + return None + host = proxy_config.get('host') + return host if host else None + + def _get_http_config_value(self, key: str): + """Extract a value from http_config dict.""" + if not self.scanoss_settings: + return None + config = self.scanoss_settings.get_http_config() + return config.get(key) if config else None + + def _get_file_snippet_http_config_value(self, key: str): + """Extract a value from file_snippet http_config dict.""" + if not self.scanoss_settings: + return None + config = self.scanoss_settings.get_file_snippet_http_config() + return config.get(key) if config else None \ No newline at end of file diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index db6469a7..6db4ed40 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -39,6 +39,7 @@ from . import __version__ from .csvoutput import CsvOutput from .cyclonedx import CycloneDx +from .scan_settings_builder import ScanSettingsBuilder from .scancodedeps import ScancodeDeps from .scanoss_settings import ScanossSettings from .scanossapi import ScanossApi @@ -103,9 +104,14 @@ def __init__( # noqa: PLR0913, PLR0915 strip_hpsm_ids=None, strip_snippet_ids=None, skip_md5_ids=None, - scan_settings: 'ScanossSettings | None' = None, + scanoss_settings: 'ScanossSettings | None' = None, req_headers: dict = None, use_grpc: bool = False, + min_snippet_hits: int = None, + min_snippet_lines: int = None, + ranking: str = None, + ranking_threshold: int = None, + honour_file_exts: str = None, skip_headers: bool = False, skip_headers_limit: int = 0, wfp_output: str = None, @@ -132,8 +138,20 @@ def __init__( # noqa: PLR0913, PLR0915 self.skip_size = skip_size self.skip_extensions = skip_extensions self.req_headers = req_headers + self.scanoss_settings = scanoss_settings ver_details = Scanner.version_details() + # Get settings values for skip_headers options + file_snippet_settings = scanoss_settings.get_file_snippet_settings() if scanoss_settings else {} + settings_skip_headers = file_snippet_settings.get('skip_headers') + settings_skip_headers_limit = file_snippet_settings.get('skip_headers_limit') + + # Merge CLI values with settings (scanoss.json takes priority over CLI) + skip_headers = Scanner._merge_cli_with_settings(skip_headers, settings_skip_headers) + skip_headers_limit = Scanner._merge_cli_with_settings( + skip_headers_limit, settings_skip_headers_limit) + self.print_debug(f'Skip headers {skip_headers} with limit: {skip_headers_limit}') + self.winnowing = Winnowing( debug=debug, trace=trace, @@ -148,21 +166,40 @@ def __init__( # noqa: PLR0913, PLR0915 skip_headers=skip_headers, skip_headers_limit=skip_headers_limit, ) + + # Build merged settings using builder pattern + scan_settings = (ScanSettingsBuilder(scanoss_settings, debug=debug, trace=trace, quiet=quiet) + .with_proxy(proxy) + .with_url(url) + .with_ignore_cert_errors(ignore_cert_errors) + .with_min_snippet_hits(min_snippet_hits) + .with_min_snippet_lines(min_snippet_lines) + .with_honour_file_exts(honour_file_exts) + .with_ranking(ranking) + .with_ranking_threshold(ranking_threshold)) + + self.print_debug(f'Scan settings: {scan_settings}') + self.scanoss_api = ScanossApi( debug=debug, trace=trace, quiet=quiet, api_key=api_key, - url=url, + url=scan_settings.url, flags=flags, timeout=timeout, ver_details=ver_details, - ignore_cert_errors=ignore_cert_errors, - proxy=proxy, + ignore_cert_errors=scan_settings.ignore_cert_errors, + proxy=scan_settings.proxy, ca_cert=ca_cert, pac=pac, retry=retry, - req_headers= self.req_headers, + req_headers=self.req_headers, + min_snippet_hits=scan_settings.min_snippet_hits, + min_snippet_lines=scan_settings.min_snippet_lines, + honour_file_exts=scan_settings.honour_file_exts, + ranking=scan_settings.ranking, + ranking_threshold=scan_settings.ranking_threshold, ) sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) grpc_api = ScanossGrpc( @@ -193,19 +230,32 @@ def __init__( # noqa: PLR0913, PLR0915 if self._skip_snippets: self.max_post_size = 8 * 1024 # 8k Max post size if we're skipping snippets - self.scan_settings = scan_settings self.post_processor = ( - ScanPostProcessor(scan_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None + ScanPostProcessor(scanoss_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None ) self._maybe_set_api_sbom() def _maybe_set_api_sbom(self): - if not self.scan_settings: + if not self.scanoss_settings: return - sbom = self.scan_settings.get_sbom() + sbom = self.scanoss_settings.get_sbom() if sbom: self.scanoss_api.set_sbom(sbom) + @staticmethod + def _merge_cli_with_settings(cli_value, settings_value): + """Merge CLI value with settings value (two-level priority: settings > cli). + + Args: + cli_value: Value from CLI argument + settings_value: Value from scanoss.json file_snippet settings + Returns: + Merged value with CLI taking priority over settings + """ + if settings_value is not None: + return settings_value + return cli_value + @staticmethod def __count_files_in_wfp_file(wfp_file: str): """ @@ -288,7 +338,8 @@ def is_dependency_scan(self): """ if self.scan_options & ScanType.SCAN_DEPENDENCIES.value: return True - return False + file_snippet_settings = self.scanoss_settings.get_file_snippet_settings() if self.scanoss_settings else {} + return file_snippet_settings.get('dependency_analysis', False) def scan_folder_with_options( # noqa: PLR0913 self, @@ -356,7 +407,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 debug=self.debug, trace=self.trace, quiet=self.quiet, - scanoss_settings=self.scan_settings, + scanoss_settings=self.scanoss_settings, all_extensions=self.all_extensions, all_folders=self.all_folders, hidden_files_folders=self.hidden_files_folders, @@ -624,7 +675,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 debug=self.debug, trace=self.trace, quiet=self.quiet, - scanoss_settings=self.scan_settings, + scanoss_settings=self.scanoss_settings, all_extensions=self.all_extensions, all_folders=self.all_folders, hidden_files_folders=self.hidden_files_folders, @@ -939,7 +990,7 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None): debug=self.debug, trace=self.trace, quiet=self.quiet, - scanoss_settings=self.scan_settings, + scanoss_settings=self.scanoss_settings, all_extensions=self.all_extensions, all_folders=self.all_folders, hidden_files_folders=self.hidden_files_folders, diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index ff9b9292..02ab9f32 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -335,3 +335,99 @@ def get_skip_sizes(self, operation_type: str) -> List[SizeFilter]: List: Min and max sizes to skip """ return self.data.get('settings', {}).get('skip', {}).get('sizes', {}).get(operation_type, []) + + def get_file_snippet_settings(self) -> dict: + """ + Get the file_snippet settings section + Returns: + dict: File snippet settings + """ + return self.data.get('settings', {}).get('file_snippet', {}) + + def get_min_snippet_hits(self) -> Optional[int]: + """ + Get the minimum snippet hits required + Returns: + int or None: Minimum snippet hits, or None if not set + """ + return self.get_file_snippet_settings().get('min_snippet_hits') + + def get_min_snippet_lines(self) -> Optional[int]: + """ + Get the minimum snippet lines required + Returns: + int or None: Minimum snippet lines, or None if not set + """ + return self.get_file_snippet_settings().get('min_snippet_lines') + + def get_ranking_enabled(self) -> Optional[bool]: + """ + Get whether ranking is enabled + Returns: + bool or None: True if enabled, False if disabled, None if not set + """ + return self.get_file_snippet_settings().get('ranking_enabled') + + def get_ranking_threshold(self) -> Optional[int]: + """ + Get the ranking threshold value + Returns: + int or None: Ranking threshold, or None if not set + """ + return self.get_file_snippet_settings().get('ranking_threshold') + + def get_honour_file_exts(self) -> Optional[bool]: + """ + Get whether to honour file extensions + Returns: + bool or None: True to honour, False to ignore, None if not set + """ + return self.get_file_snippet_settings().get('honour_file_exts') + + def get_skip_headers_limit(self) -> int: + """ + Get the skip headers limit value + Returns: + int: Skip headers limit, or 0 if not set + """ + return self.get_file_snippet_settings().get('skip_headers_limit', 0) + + def get_skip_headers(self) -> bool: + """ + Get whether to skip headers + Returns: + bool: True to skip headers, False otherwise (default) + """ + return self.get_file_snippet_settings().get('skip_headers', False) + + def get_proxy(self) -> Optional[dict]: + """ + Get the root-level proxy configuration + Returns: + dict or None: Proxy configuration with 'host' key, or None if not set + """ + return self.data.get('settings', {}).get('proxy') + + def get_http_config(self) -> Optional[dict]: + """ + Get the root-level http_config configuration + Returns: + dict or None: HTTP config with 'base_uri' and 'ignore_cert_errors' keys, or None if not set + """ + return self.data.get('settings', {}).get('http_config') + + def get_file_snippet_proxy(self) -> Optional[dict]: + """ + Get the file_snippet-level proxy configuration (takes priority over root) + Returns: + dict or None: Proxy configuration with 'host' key, or None if not set + """ + return self.get_file_snippet_settings().get('proxy') + + def get_file_snippet_http_config(self) -> Optional[dict]: + """ + Get the file_snippet-level http_config configuration (takes priority over root) + Returns: + dict or None: HTTP config with 'base_uri' and 'ignore_cert_errors' keys, or None if not set + """ + return self.get_file_snippet_settings().get('http_config') diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 1e194da4..73f0838a 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -22,13 +22,16 @@ THE SOFTWARE. """ +import base64 import http.client as http_client +import json import logging import os import sys import time import uuid from json.decoder import JSONDecodeError +from typing import Optional, Union from urllib.parse import urlparse, urlunparse import requests @@ -98,6 +101,11 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 pac: PACFile = None, retry: int = 5, req_headers: dict = None, + min_snippet_hits: int = None, + min_snippet_lines: int = None, + honour_file_exts: Union[bool, str, None] = 'unset', + ranking: Union[bool, str, None] = 'unset', + ranking_threshold: int = None, ): """ Initialise the SCANOSS API @@ -108,6 +116,11 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 :param debug: Enable debug (default False) :param trace: Enable trace (default False) :param quiet: Enable quiet mode (default False) + :param min_snippet_hits: Minimum snippet hits required (default None) + :param min_snippet_lines: Minimum snippet lines required (default None) + :param honour_file_exts: Whether to honour file extensions (default 'unset') + :param ranking: Enable/disable ranking (default 'unset') + :param ranking_threshold: Ranking threshold value (default None) To set a custom certificate use: REQUESTS_CA_BUNDLE=/path/to/cert.pem @@ -117,6 +130,12 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 """ super().__init__(debug, trace, quiet) self.sbom = None + # Scan tuning parameters + self.min_snippet_hits = min_snippet_hits + self.min_snippet_lines = min_snippet_lines + self.honour_file_exts = honour_file_exts + self.ranking = ranking + self.ranking_threshold = ranking_threshold self.scan_format = scan_format if scan_format else 'plain' self.flags = flags self.timeout = timeout if timeout > MIN_TIMEOUT else DEFAULT_TIMEOUT @@ -183,6 +202,10 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None): # noqa: PLR scan_files = {'file': ('%s.wfp' % request_id, wfp)} headers = self.headers headers['x-request-id'] = request_id # send a unique request id for each post + # Add scan settings header if any settings are configured + scan_settings_header = self._build_scan_settings_header() + if scan_settings_header: + headers['scanoss-settings'] = scan_settings_header r = None retry = 0 # Add some retry logic to cater for timeouts, etc. while retry <= self.retry_limit: @@ -296,6 +319,41 @@ def set_sbom(self, sbom): self.sbom = sbom return self + def _build_scan_settings_header(self) -> Optional[str]: + """ + Build base64-encoded JSON for x-scanoss-scan-settings header. + Only includes parameters that have meaningful (non-"unset") values. + Returns: + Base64-encoded JSON string, or None if no settings to send + """ + settings = {} + + # min_snippet_hits: 0 = unset, don't send + if self.min_snippet_hits is not None and self.min_snippet_hits != 0: + settings['min_snippet_hits'] = self.min_snippet_hits + + # min_snippet_lines: 0 = unset, don't send + if self.min_snippet_lines is not None and self.min_snippet_lines != 0: + settings['min_snippet_lines'] = self.min_snippet_lines + + # honour_file_exts: None = unset, don't send + if self.honour_file_exts is not None and self.honour_file_exts != 'unset': + settings['honour_file_exts'] = self.honour_file_exts + + # ranking: None = unset, don't send + if self.ranking is not None and self.ranking != 'unset': + settings['ranking_enabled'] = self.ranking + + # ranking_threshold: -1 = unset, don't send + if self.ranking_threshold is not None and self.ranking_threshold != -1: + settings['ranking_threshold'] = self.ranking_threshold + + if settings: + json_str = json.dumps(settings) + self.print_debug(f'Scan settings: {json_str}') + return base64.b64encode(json_str.encode()).decode() + return None + def load_generic_headers(self, url): """ Adds custom headers from req_headers to the headers collection. diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 4dd6f85d..b86accc9 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -80,7 +80,7 @@ class ScanPostProcessor(ScanossBase): def __init__( self, - scan_settings: ScanossSettings, + scanoss_settings: ScanossSettings, debug: bool = False, trace: bool = False, quiet: bool = False, @@ -88,14 +88,14 @@ def __init__( ): """ Args: - scan_settings (ScanossSettings): Scan settings object + scanoss_settings (ScanossSettings): Scanoss settings object debug (bool, optional): Debug mode. Defaults to False. trace (bool, optional): Traces. Defaults to False. quiet (bool, optional): Quiet mode. Defaults to False. results (dict | str, optional): Results to be processed. Defaults to None. """ super().__init__(debug, trace, quiet) - self.scan_settings = scan_settings + self.scanoss_settings = scanoss_settings self.results: dict = results self.component_info_map: dict = {} @@ -114,10 +114,10 @@ def _load_component_info(self): if not self.results: return for _, result in self.results.items(): - result = result[0] if isinstance(result, list) else result - purls = result.get('purl', []) + entry = result[0] if isinstance(result, list) else result + purls = entry.get('purl', []) for purl in purls: - self.component_info_map[purl] = result + self.component_info_map[purl] = entry def post_process(self): """ @@ -126,7 +126,7 @@ def post_process(self): Returns: dict: Processed results """ - if self.scan_settings.is_legacy(): + if self.scanoss_settings.is_legacy(): self.print_stderr( 'Legacy settings file detected. Post-processing is not supported for legacy settings file.' ) @@ -139,7 +139,7 @@ def _remove_dismissed_files(self): """ Remove entries from the results based on files and/or purls specified in the SCANOSS settings file """ - to_remove_entries = self.scan_settings.get_bom_remove() + to_remove_entries = self.scanoss_settings.get_bom_remove() if not to_remove_entries: return self.results = { @@ -152,15 +152,15 @@ def _replace_purls(self): """ Replace purls in the results based on the SCANOSS settings file """ - to_replace_entries = self.scan_settings.get_bom_replace() + to_replace_entries = self.scanoss_settings.get_bom_replace() if not to_replace_entries: return for result_path, result in self.results.items(): - result = result[0] if isinstance(result, list) else result - should_replace, to_replace_with_purl = self._should_replace_result(result_path, result, to_replace_entries) + entry = result[0] if isinstance(result, list) else result + should_replace, to_replace_with_purl = self._should_replace_result(result_path, entry, to_replace_entries) if should_replace: - self.results[result_path] = [self._update_replaced_result(result, to_replace_with_purl)] + self.results[result_path] = [self._update_replaced_result(entry, to_replace_with_purl)] def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> dict: """ diff --git a/tests/data/scanoss.json b/tests/data/scanoss.json index b33ddcf1..fac0b807 100644 --- a/tests/data/scanoss.json +++ b/tests/data/scanoss.json @@ -25,6 +25,29 @@ "replace_with": "pkg:github/scanoss/only_purl_match_replaced.py" } ] + }, + "settings": { + "proxy": { + "host": "http://root-proxy:8080" + }, + "http_config": { + "base_uri": "https://root-api.scanoss.com", + "ignore_cert_errors": false + }, + "file_snippet": { + "proxy": { + "host": "http://file-snippet-proxy:8080" + }, + "http_config": { + "base_uri": "https://file-snippet-api.scanoss.com", + "ignore_cert_errors": true + }, + "min_snippet_hits": 10, + "min_snippet_lines": 5, + "honour_file_exts": true, + "ranking_enabled": true, + "ranking_threshold": 10 + } } } diff --git a/tests/scanossapi-test.py b/tests/scanossapi-test.py index a9fa1db1..e28e4556 100644 --- a/tests/scanossapi-test.py +++ b/tests/scanossapi-test.py @@ -30,7 +30,7 @@ class MyTestCase(unittest.TestCase): def test_scanoss_generic_headers(self): scanoss_api = ScanossApi(debug=True, req_headers={'x-api-key': '123455', 'generic-header': 'generic-header-value'}) - required_keys = ('x-api-key', 'User-Agent', 'user-agent', 'generic-header') + required_keys = ('x-api-key', 'X-Session', 'User-Agent', 'user-agent', 'generic-header') valid_headers = True for key, value in scanoss_api.headers.items(): if key not in required_keys: diff --git a/tests/test_scan_settings_builder.py b/tests/test_scan_settings_builder.py new file mode 100644 index 00000000..4b73820b --- /dev/null +++ b/tests/test_scan_settings_builder.py @@ -0,0 +1,362 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2025, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import os +import unittest +from pathlib import Path + +from src.scanoss.scan_settings_builder import (ScanSettingsBuilder) +from src.scanoss.scanoss_settings import ScanossSettings + + +class TestScanSettingsBuilder(unittest.TestCase): + """Tests for the ScanSettingsBuilder class.""" + + script_dir = os.path.dirname(os.path.abspath(__file__)) + scan_settings_path = Path(script_dir, 'data', 'scanoss.json').resolve() + scan_settings = ScanossSettings(filepath=scan_settings_path) + + # ========================================================================= + # Test initialization + # ========================================================================= + + def test_init_with_none_settings(self): + """Test initialization with None settings.""" + builder = ScanSettingsBuilder(None) + + self.assertIsNone(builder.scanoss_settings) + self.assertIsNone(builder.proxy) + self.assertIsNone(builder.url) + self.assertFalse(builder.ignore_cert_errors) + self.assertIsNone(builder.min_snippet_hits) + self.assertIsNone(builder.min_snippet_lines) + self.assertIsNone(builder.honour_file_exts) + self.assertIsNone(builder.ranking) + self.assertIsNone(builder.ranking_threshold) + + def test_init_with_settings(self): + """Test initialization with settings object.""" + builder = ScanSettingsBuilder(self.scan_settings) + + self.assertEqual(builder.scanoss_settings, self.scan_settings) + + # ========================================================================= + # Test static helper methods + # ========================================================================= + + def test_str_to_bool_with_none(self): + """Test _str_to_bool returns None for None input.""" + self.assertIsNone(ScanSettingsBuilder._str_to_bool(None)) + + def test_str_to_bool_with_true_string(self): + """Test _str_to_bool converts 'true' to True.""" + self.assertTrue(ScanSettingsBuilder._str_to_bool('true')) + self.assertTrue(ScanSettingsBuilder._str_to_bool('True')) + self.assertTrue(ScanSettingsBuilder._str_to_bool('TRUE')) + + def test_str_to_bool_with_false_string(self): + """Test _str_to_bool converts 'false' to False.""" + self.assertFalse(ScanSettingsBuilder._str_to_bool('false')) + self.assertFalse(ScanSettingsBuilder._str_to_bool('False')) + self.assertFalse(ScanSettingsBuilder._str_to_bool('FALSE')) + + def test_str_to_bool_with_bool_input(self): + """Test _str_to_bool passes through bool values.""" + self.assertTrue(ScanSettingsBuilder._str_to_bool(True)) + self.assertFalse(ScanSettingsBuilder._str_to_bool(False)) + + def test_merge_with_priority_file_snippet_wins(self): + """Test _merge_with_priority returns file_snippet value when present (highest priority).""" + result = ScanSettingsBuilder._merge_with_priority('cli', 'file_snippet', 'root') + self.assertEqual(result, 'file_snippet') + + def test_merge_with_priority_root_second(self): + """Test _merge_with_priority returns root when file_snippet is None.""" + result = ScanSettingsBuilder._merge_with_priority('cli', None, 'root') + self.assertEqual(result, 'root') + + def test_merge_with_priority_cli_fallback(self): + """Test _merge_with_priority returns CLI when others are None.""" + result = ScanSettingsBuilder._merge_with_priority('cli', None, None) + self.assertEqual(result, 'cli') + + def test_merge_with_priority_all_none(self): + """Test _merge_with_priority returns None when all are None.""" + result = ScanSettingsBuilder._merge_with_priority(None, None, None) + self.assertIsNone(result) + + def test_merge_cli_with_settings_settings_wins(self): + """Test _merge_cli_with_settings returns settings value when present (highest priority).""" + result = ScanSettingsBuilder._merge_cli_with_settings('cli', 'settings') + self.assertEqual(result, 'settings') + + def test_merge_cli_with_settings_cli_fallback(self): + """Test _merge_cli_with_settings returns CLI when settings is None.""" + result = ScanSettingsBuilder._merge_cli_with_settings('cli', None) + self.assertEqual(result, 'cli') + + # ========================================================================= + # Test with_proxy + # ========================================================================= + + def test_with_proxy_cli_only(self): + """Test with_proxy uses CLI value when no settings.""" + builder = ScanSettingsBuilder(None) + result = builder.with_proxy('http://cli-proxy:8080') + + self.assertEqual(builder.proxy, 'http://cli-proxy:8080') + self.assertEqual(result, builder) # Test chaining + + def test_with_proxy_from_file_snippet(self): + """Test with_proxy uses file_snippet.proxy.host when CLI is None.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_proxy(None) + + # file_snippet.proxy.host = "http://file-snippet-proxy:8080" + self.assertEqual(builder.proxy, 'http://file-snippet-proxy:8080') + + def test_with_proxy_settings_overrides_cli(self): + """Test with_proxy settings value overrides CLI.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_proxy('http://cli-proxy:8080') + + # file_snippet.proxy.host = "http://file-snippet-proxy:8080" takes priority + self.assertEqual(builder.proxy, 'http://file-snippet-proxy:8080') + + # ========================================================================= + # Test with_url + # ========================================================================= + + def test_with_url_cli_only(self): + """Test with_url uses CLI value when no settings.""" + builder = ScanSettingsBuilder(None) + builder.with_url('https://cli-api.example.com') + + self.assertEqual(builder.url, 'https://cli-api.example.com') + + def test_with_url_from_file_snippet(self): + """Test with_url uses file_snippet.http_config.base_uri.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_url(None) + + # file_snippet.http_config.base_uri = "https://file-snippet-api.scanoss.com" + self.assertEqual(builder.url, 'https://file-snippet-api.scanoss.com') + + def test_with_url_settings_overrides_cli(self): + """Test with_url settings value overrides CLI.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_url('https://cli-api.com') + + # file_snippet.http_config.base_uri = "https://file-snippet-api.scanoss.com" takes priority + self.assertEqual(builder.url, 'https://file-snippet-api.scanoss.com') + + # ========================================================================= + # Test with_ignore_cert_errors + # ========================================================================= + + def test_with_ignore_cert_errors_defaults_to_false(self): + """Test with_ignore_cert_errors defaults to False.""" + builder = ScanSettingsBuilder(None) + builder.with_ignore_cert_errors(False) + + self.assertFalse(builder.ignore_cert_errors) + + def test_with_ignore_cert_errors_cli_true(self): + """Test with_ignore_cert_errors with CLI True.""" + builder = ScanSettingsBuilder(None) + builder.with_ignore_cert_errors(True) + + self.assertTrue(builder.ignore_cert_errors) + + def test_with_ignore_cert_errors_from_file_snippet(self): + """Test with_ignore_cert_errors from file_snippet settings.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_ignore_cert_errors(False) + + # file_snippet.http_config.ignore_cert_errors = true + self.assertTrue(builder.ignore_cert_errors) + + def test_with_ignore_cert_errors_cli_true_overrides(self): + """Test with_ignore_cert_errors CLI True overrides settings.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_ignore_cert_errors(True) + + self.assertTrue(builder.ignore_cert_errors) + + # ========================================================================= + # Test with_min_snippet_hits + # ========================================================================= + + def test_with_min_snippet_hits_cli_only(self): + """Test with_min_snippet_hits uses CLI value.""" + builder = ScanSettingsBuilder(None) + builder.with_min_snippet_hits(5) + + self.assertEqual(builder.min_snippet_hits, 5) + + def test_with_min_snippet_hits_from_settings(self): + """Test with_min_snippet_hits from settings.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_min_snippet_hits(None) + + # file_snippet.min_snippet_hits = 10 + self.assertEqual(builder.min_snippet_hits, 10) + + def test_with_min_snippet_hits_settings_overrides_cli(self): + """Test with_min_snippet_hits settings overrides CLI.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_min_snippet_hits(5) + + # file_snippet.min_snippet_hits = 10 takes priority + self.assertEqual(builder.min_snippet_hits, 10) + + # ========================================================================= + # Test with_min_snippet_lines + # ========================================================================= + + def test_with_min_snippet_lines_cli_only(self): + """Test with_min_snippet_lines uses CLI value.""" + builder = ScanSettingsBuilder(None) + builder.with_min_snippet_lines(3) + + self.assertEqual(builder.min_snippet_lines, 3) + + def test_with_min_snippet_lines_from_settings(self): + """Test with_min_snippet_lines from settings.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_min_snippet_lines(None) + + # file_snippet.min_snippet_lines = 5 + self.assertEqual(builder.min_snippet_lines, 5) + + # ========================================================================= + # Test with_honour_file_exts + # ========================================================================= + + def test_with_honour_file_exts_cli_true(self): + """Test with_honour_file_exts with CLI 'true'.""" + builder = ScanSettingsBuilder(None) + builder.with_honour_file_exts('true') + + self.assertTrue(builder.honour_file_exts) + + def test_with_honour_file_exts_cli_false(self): + """Test with_honour_file_exts with CLI 'false'.""" + builder = ScanSettingsBuilder(None) + builder.with_honour_file_exts('false') + + self.assertFalse(builder.honour_file_exts) + + def test_with_honour_file_exts_from_settings(self): + """Test with_honour_file_exts from settings.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_honour_file_exts(None) + + # file_snippet.honour_file_exts = true + self.assertTrue(builder.honour_file_exts) + + def test_with_honour_file_exts_settings_overrides_cli(self): + """Test with_honour_file_exts settings overrides CLI.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_honour_file_exts('false') + + # file_snippet.honour_file_exts = true takes priority + self.assertTrue(builder.honour_file_exts) + + # ========================================================================= + # Test with_ranking + # ========================================================================= + + def test_with_ranking_cli_true(self): + """Test with_ranking with CLI 'true'.""" + builder = ScanSettingsBuilder(None) + builder.with_ranking('true') + + self.assertTrue(builder.ranking) + + def test_with_ranking_cli_false(self): + """Test with_ranking with CLI 'false'.""" + builder = ScanSettingsBuilder(None) + builder.with_ranking('false') + + self.assertFalse(builder.ranking) + + def test_with_ranking_from_settings(self): + """Test with_ranking from settings.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_ranking(None) + + # file_snippet.ranking_enabled = true + self.assertTrue(builder.ranking) + + # ========================================================================= + # Test with_ranking_threshold + # ========================================================================= + + def test_with_ranking_threshold_cli_only(self): + """Test with_ranking_threshold uses CLI value.""" + builder = ScanSettingsBuilder(None) + builder.with_ranking_threshold(50) + + self.assertEqual(builder.ranking_threshold, 10) + + def test_with_ranking_threshold_from_settings(self): + """Test with_ranking_threshold from settings.""" + builder = ScanSettingsBuilder(self.scan_settings) + builder.with_ranking_threshold(None) + + # file_snippet.ranking_threshold = 10 + self.assertEqual(builder.ranking_threshold, 10) + + # ========================================================================= + # Test method chaining + # ========================================================================= + + def test_method_chaining(self): + """Test that all with_* methods support chaining.""" + builder = ScanSettingsBuilder(None) + + result = (builder + .with_proxy('http://proxy:8080') + .with_url('https://api.example.com') + .with_ignore_cert_errors(True) + .with_min_snippet_hits(5) + .with_min_snippet_lines(3) + .with_honour_file_exts('true') + .with_ranking('true') + .with_ranking_threshold(10)) + + self.assertEqual(result, builder) + self.assertEqual(builder.proxy, 'http://proxy:8080') + self.assertEqual(builder.url, 'https://api.example.com') + self.assertTrue(builder.ignore_cert_errors) + self.assertEqual(builder.min_snippet_hits, 5) + self.assertEqual(builder.min_snippet_lines, 3) + self.assertTrue(builder.honour_file_exts) + self.assertTrue(builder.ranking) + self.assertEqual(builder.ranking_threshold, 10) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 7d91b337bdf86ff09edb54ed368ca61a33ee04a5 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Mon, 2 Feb 2026 11:21:47 -0300 Subject: [PATCH 419/489] chore(version): update __init__.py version --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index b7db689a..005593e1 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.44.0' +__version__ = '1.45.0' From 6123b314adc372a494c74016f13e65d94e45b763 Mon Sep 17 00:00:00 2001 From: Andrei Sacal Date: Wed, 4 Feb 2026 16:48:55 +0100 Subject: [PATCH 420/489] Fix default ranking_threshold value in scanoss-settings-schema.json --- src/scanoss/data/scanoss-settings-schema.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/scanoss/data/scanoss-settings-schema.json b/src/scanoss/data/scanoss-settings-schema.json index 124f9c66..13672c94 100644 --- a/src/scanoss/data/scanoss-settings-schema.json +++ b/src/scanoss/data/scanoss-settings-schema.json @@ -177,7 +177,7 @@ "type": ["integer", "null"], "description": "Ranking threshold value. A value of -1 defers to server configuration", "minimum": -1, - "maximum": 99, + "maximum": 10, "default": 0 }, "min_snippet_hits": { @@ -345,4 +345,3 @@ } } } - From f0ccd9ba36bc8c5d8bc1eb66e6dad31a0154088d Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Mon, 9 Feb 2026 11:53:12 -0300 Subject: [PATCH 421/489] chore(doc):SP-4029 update scanoss_settings_schema.rst with scan tunning parameters --- docs/source/scanoss_settings_schema.rst | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index e24cfb26..decebbc5 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -193,6 +193,59 @@ Examples with Explanations +Scan Tuning Parameters +---------------------- +The SCANOSS scan engine supports tuning parameters for snippet matching. These parameters allow you to fine-tune how the scanner identifies code snippets in your repository. + +.. list-table:: + :header-rows: 1 + :widths: 20 15 10 55 + + * - Parameter + - Type + - Default + - Description + * - ``min_snippet_hits`` + - ``integer`` + - ``0`` + - Minimum snippet hits required. ``0`` defers to server configuration. + * - ``min_snippet_lines`` + - ``integer`` + - ``0`` + - Minimum snippet lines required. ``0`` defers to server configuration. + * - ``ranking_enabled`` + - ``boolean | null`` + - ``null`` + - Enable/disable result ranking. ``null`` defers to server configuration. + * - ``ranking_threshold`` + - ``integer | null`` + - ``0`` + - Ranking threshold value (``-1`` to ``10``). ``-1`` defers to server configuration. + * - ``honour_file_exts`` + - ``boolean | null`` + - ``true`` + - Honour file extensions during matching. ``null`` defers to server configuration. + +Example Configuration +~~~~~~~~~~~~~~~~~~~~~ + +Add the ``file_snippet`` section to your ``scanoss.json`` file: + +.. code-block:: json + + { + "settings": { + "file_snippet": { + "min_snippet_hits": 3, + "min_snippet_lines": 5, + "ranking_enabled": true, + "ranking_threshold": 5, + "honour_file_exts": true + } + } + } + + Complete Example ------------------- Here's a comprehensive example combining pattern and size-based skipping: @@ -420,6 +473,13 @@ Here's a complete example showing all sections: } ] } + }, + "file_snippet": { + "min_snippet_hits": 3, + "min_snippet_lines": 5, + "ranking_enabled": true, + "ranking_threshold": 5, + "honour_file_exts": true } }, "bom": { From 69fee4076a6db4c898a71ffefc19f702a72cf8a1 Mon Sep 17 00:00:00 2001 From: Gepeto Escalante Date: Sat, 14 Feb 2026 08:09:21 +0000 Subject: [PATCH 422/489] fix(inspect):SP-3728 Fix scanoss-py inspect help display --- CHANGELOG.md | 5 +++++ src/scanoss/cli.py | 27 ++------------------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4166b69f..2b6d6e3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... +## [1.45.1] - 2026-02-23 +### Fixed +- Fixed `--input` argument validation for inspect subcommands (copyleft, undeclared, license-summary, component-summary) by making it required at the argparse level instead of manual runtime checks + ## [1.45.0] - 2026-02-02 ### Added - Added scan engine tuning parameters for snippet matching: @@ -796,3 +800,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.43.1]: https://github.com/scanoss/scanoss.py/compare/v1.43.0...v1.43.1 [1.44.0]: https://github.com/scanoss/scanoss.py/compare/v1.43.1...v1.44.0 [1.45.0]: https://github.com/scanoss/scanoss.py/compare/v1.44.0...v1.45.0 +[1.45.1]: https://github.com/scanoss/scanoss.py/compare/v1.45.0...v1.45.1 \ No newline at end of file diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 93bc5011..8216d85e 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -755,7 +755,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 # Common options for (legacy) copyleft and undeclared component inspection for p in [p_inspect_raw_copyleft, p_inspect_raw_undeclared, p_inspect_legacy_copyleft, p_inspect_legacy_undeclared]: - p.add_argument('-i', '--input', nargs='?', help='Path to scan results file to analyse') + p.add_argument('-i', '--input', required=True, help='Path to scan results file to analyse') p.add_argument( '-f', '--format', @@ -774,7 +774,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_inspect_legacy_license_summary, p_inspect_legacy_component_summary, ]: - p.add_argument('-i', '--input', nargs='?', help='Path to scan results file to analyse') + p.add_argument('-i', '--input', required=True, help='Path to scan results file to analyse') p.add_argument('-o', '--output', type=str, help='Save summary report to specified file') # ------------------------------------------------------------------------- @@ -1803,11 +1803,6 @@ def inspect_copyleft(parser, args): - format: Output format (json, md, jira_md) - include/exclude/explicit: License filter options """ - # Validate required input file parameter - if args.input is None: - print_stderr('ERROR: Input file is required for copyleft inspection') - parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) - sys.exit(1) # Initialise output file if specified if args.output: initialise_empty_file(args.output) @@ -1859,12 +1854,6 @@ def inspect_undeclared(parser, args): - format: Output format (json, md, jira_md) - sbom_format: SBOM format type (legacy, settings) """ - # Validate required input file parameter - if args.input is None: - print_stderr('ERROR: Input file is required for undeclared component inspection') - parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) - sys.exit(1) - # Initialise output file if specified if args.output: initialise_empty_file(args.output) @@ -1913,12 +1902,6 @@ def inspect_license_summary(parser, args): - output: Optional output file path - include/exclude/explicit: License filter options """ - # Validate required input file parameter - if args.input is None: - print_stderr('ERROR: Input file is required for license summary') - parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) - sys.exit(1) - # Initialise output file if specified if args.output: initialise_empty_file(args.output) @@ -1960,12 +1943,6 @@ def inspect_component_summary(parser, args): - input: Path to scan results file - output: Optional output file path """ - # Validate required input file parameter - if args.input is None: - print_stderr('ERROR: Input file is required for component summary') - parser.parse_args([args.subparser, args.subparsercmd, args.subparser_subcmd, '-h']) - sys.exit(1) - # Initialise an output file if specified if args.output: initialise_empty_file(args.output) # Create/clear output file From 4b395f1d496416f2f174e8a2c0a4ed0c1ec39652 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Mon, 23 Feb 2026 08:15:19 -0300 Subject: [PATCH 423/489] chore(version): upgrade version to v1.45.1 --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 005593e1..8a47bde0 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.45.0' +__version__ = '1.45.1' From 6e136199838298c531f35e539acdaabd31edfb89 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 23 Feb 2026 18:34:01 +0100 Subject: [PATCH 424/489] docs(schema): update references to scanoss settings schema --- docs/source/_static/scanoss-settings-schema.json | 1 + docs/source/scanoss_settings_schema.rst | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/source/_static/scanoss-settings-schema.json b/docs/source/_static/scanoss-settings-schema.json index 7cabb8ef..153ba9d2 100644 --- a/docs/source/_static/scanoss-settings-schema.json +++ b/docs/source/_static/scanoss-settings-schema.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema", + "$comment": "DEPRECATED: The canonical schema is now maintained at https://github.com/scanoss/schema/blob/main/scanoss-settings-schema.json", "title": "Scanoss Settings", "type": "object", "properties": { diff --git a/docs/source/scanoss_settings_schema.rst b/docs/source/scanoss_settings_schema.rst index decebbc5..14afa9d9 100644 --- a/docs/source/scanoss_settings_schema.rst +++ b/docs/source/scanoss_settings_schema.rst @@ -1,6 +1,13 @@ Settings File ====================== +.. warning:: **Deprecated** — This documentation is no longer maintained here. + The settings schema and its documentation have moved to the + `scanoss/schema `_ repository. + Please refer to the `interactive docs `_ + or the `canonical JSON Schema `_ + for the latest version. + SCANOSS provides a settings file to customize the scanning process. The settings file is a JSON file that contains project information and BOM (Bill of Materials) rules. It allows you to include, remove, or replace components in the BOM before and after scanning. The schema is available to download :download:`here ` From bd6eb4aea6c9d122465d65f0de888f96d6109cec Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 11:52:17 +0100 Subject: [PATCH 425/489] feat(settings): add folder-level path matching and priority resolution for BOM rules Add path matching helpers that support three levels: - Purl-only (no path): applies globally - File-level (exact path): applies to specific file - Folder-level (trailing '/'): applies to all files under directory Add priority-based rule resolution: path+purl (4) > purl-only (2) > path-only (1), with longer paths winning on equal priority. Update JSON schema to allow path-only remove entries (anyOf purl/path) and add folder path examples to include/remove/replace sections. Add per-batch SBOM context resolution (get_sbom_for_batch) for scoping include/exclude purls to requests containing matching files. --- src/scanoss/data/scanoss-settings-schema.json | 23 ++- src/scanoss/scanoss_settings.py | 153 +++++++++++++++++- 2 files changed, 163 insertions(+), 13 deletions(-) diff --git a/src/scanoss/data/scanoss-settings-schema.json b/src/scanoss/data/scanoss-settings-schema.json index 13672c94..c103914d 100644 --- a/src/scanoss/data/scanoss-settings-schema.json +++ b/src/scanoss/data/scanoss-settings-schema.json @@ -248,12 +248,8 @@ "properties": { "path": { "type": "string", - "description": "File path", - "examples": ["/path/to/file", "/path/to/another/file"], - "items": { - "type": "string" - }, - "uniqueItems": true + "description": "File or folder path. Paths ending with '/' are treated as folder rules and match all files under that directory.", + "examples": ["src/main.c", "src/vendor/"] }, "purl": { "type": "string", @@ -280,8 +276,8 @@ "properties": { "path": { "type": "string", - "description": "File path", - "examples": ["/path/to/file", "/path/to/another/file"] + "description": "File or folder path. Paths ending with '/' are treated as folder rules and match all files under that directory.", + "examples": ["src/main.c", "src/vendor/"] }, "purl": { "type": "string", @@ -296,8 +292,11 @@ "description": "Additional notes or comments" } }, - "uniqueItems": true, - "required": ["purl"] + "anyOf": [ + {"required": ["purl"]}, + {"required": ["path"]} + ], + "uniqueItems": true } }, "replace": { @@ -308,8 +307,8 @@ "properties": { "path": { "type": "string", - "description": "File path", - "examples": ["/path/to/file", "/path/to/another/file"] + "description": "File or folder path. Paths ending with '/' are treated as folder rules and match all files under that directory.", + "examples": ["src/main.c", "src/vendor/"] }, "purl": { "type": "string", diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 02ab9f32..caf21ab0 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -42,6 +42,92 @@ class BomEntry(TypedDict, total=False): purl: str path: str + replace_with: str + comment: str + license: str + + +def matches_path(entry_path: str, result_path: str) -> bool: + """ + Check if a BOM entry path matches a result path. + Folder paths (ending with '/') use prefix matching; file paths use exact matching. + + Args: + entry_path: Path from the BOM entry + result_path: Path from the scan result + + Returns: + True if the entry path matches the result path + """ + if not entry_path: + return True + if entry_path.endswith('/'): + return result_path.startswith(entry_path) + return entry_path == result_path + + +def entry_priority(entry: BomEntry) -> int: + """ + Calculate the priority score for a BOM entry. + Higher score means higher priority (more specific). + + Score 4: both path and purl (most specific) + Score 2: purl only + Score 1: path only (remove only, no purl) + + Args: + entry: BOM entry to evaluate + + Returns: + Priority score + """ + has_path = bool(entry.get('path')) + has_purl = bool(entry.get('purl')) + if has_path and has_purl: + return 4 + if has_purl: + return 2 + if has_path: + return 1 + return 0 + + +def find_best_match(result_path: str, result_purls: List[str], entries: List[BomEntry]) -> Optional[BomEntry]: + """ + Find the highest-priority BOM entry that matches a result. + When scores are equal, the longer path wins (more specific). + + Args: + result_path: Path from the scan result + result_purls: List of purls from the scan result + entries: List of BOM entries to check + + Returns: + The best matching BOM entry, or None if no match + """ + best_entry = None + best_score = -1 + best_path_len = -1 + + for entry in entries: + entry_path = entry.get('path', '') + entry_purl = entry.get('purl', '') + + if not entry_path and not entry_purl: + continue + if entry_path and not matches_path(entry_path, result_path): + continue + if entry_purl and (not result_purls or entry_purl not in result_purls): + continue + + score = entry_priority(entry) + path_len = len(entry_path) + if score > best_score or (score == best_score and path_len > best_path_len): + best_entry = entry + best_score = score + best_path_len = path_len + + return best_entry class SizeFilter(TypedDict, total=False): @@ -276,10 +362,75 @@ def _get_sbom_assets(self): return self.normalize_bom_entries(self.get_bom_remove()) + def has_path_scoped_bom_entries(self) -> bool: + """ + Check if any include or exclude BOM entries have path-scoped rules. + When path-scoped entries exist, the SBOM context must be resolved per-batch + instead of sent globally with every request. + + Returns: + True if any include/exclude entry has a path field + """ + for entry in self.get_bom_include() + self.get_bom_exclude(): + if entry.get('path'): + return True + return False + + def get_sbom_for_batch(self, batch_file_paths: List[str]) -> Optional[dict]: + """ + Get the SBOM context filtered for a specific batch of files. + Only includes purls from entries whose path matches files in the batch. + + Purl-only entries (no path) are always included. + File entries are included only if the exact file is in the batch. + Folder entries are included only if any file in the batch is under that folder. + + Args: + batch_file_paths: List of file paths in the current WFP batch + + Returns: + SBOM payload dict with 'assets' and 'scan_type', or None + """ + if not self.data: + return None + + include_entries = self.get_bom_include() + exclude_entries = self.get_bom_exclude() + + if not include_entries and not exclude_entries: + return None + + bom_entries = include_entries or exclude_entries + scan_type = 'identify' if include_entries else 'blacklist' + + filtered_purls = set() + for entry in bom_entries: + entry_path = entry.get('path', '') + entry_purl = entry.get('purl', '') + if not entry_purl: + continue + if not entry_path: + filtered_purls.add(entry_purl) + continue + for file_path in batch_file_paths: + if matches_path(entry_path, file_path): + filtered_purls.add(entry_purl) + break + + if not filtered_purls: + return None + + components = [{'purl': p} for p in sorted(filtered_purls)] + return { + 'assets': json.dumps({'components': components}), + 'scan_type': scan_type, + } + @staticmethod def normalize_bom_entries(bom_entries) -> List[BomEntry]: """ - Normalize the BOM entries + Normalize the BOM entries by extracting only the purl field. + Args: bom_entries (List[Dict]): List of BOM entries Returns: From b46a5a0722763f0ad0fc173c110cd70923294255 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 11:52:22 +0100 Subject: [PATCH 426/489] feat(postprocessor): use priority-based matching for remove and replace Replace first-match-wins logic with find_best_match() for both remove and replace post-processing. This enables folder-level matching (trailing '/') and ensures the most specific rule wins when multiple BOM entries match the same result. Path-only remove entries (no purl) are now supported. --- src/scanoss/scanpostprocessor.py | 94 ++++++++++---------------------- 1 file changed, 29 insertions(+), 65 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index b86accc9..999fb694 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -27,7 +27,7 @@ from packageurl import PackageURL from packageurl.contrib import purl2url -from .scanoss_settings import BomEntry, ScanossSettings +from .scanoss_settings import BomEntry, ScanossSettings, find_best_match from .scanossbase import ScanossBase @@ -43,8 +43,10 @@ def _get_match_type_message(result_path: str, bom_entry: BomEntry, action: str) Returns: str: The message to be printed """ - if bom_entry.get('path') and bom_entry.get('purl'): - message = f"{action} '{result_path}'. Full match found." + entry_path = bom_entry.get('path', '') + if entry_path and bom_entry.get('purl'): + match_kind = 'folder' if entry_path.endswith('/') else 'file' + message = f"{action} '{result_path}'. Full match found ({match_kind} + purl)." elif bom_entry.get('purl'): message = f"{action} '{result_path}'. Found PURL match." else: @@ -52,27 +54,6 @@ def _get_match_type_message(result_path: str, bom_entry: BomEntry, action: str) return message -def _is_full_match(result_path: str, result_purls: List[str], bom_entry: BomEntry) -> bool: - """ - Check if path and purl matches fully with the bom entry - - Args: - result_path (str): Scan result path - result_purls (List[str]): Scan result purls - bom_entry (BomEntry): BOM entry to compare with - - Returns: - bool: True if the path and purl match, False otherwise - """ - if not result_purls: - return False - return bool( - (bom_entry.get('purl') and bom_entry.get('path')) - and (bom_entry.get('path') == result_path) - and (bom_entry.get('purl') in result_purls) - ) - - class ScanPostProcessor(ScanossBase): """ Handles post-processing of the scan results @@ -211,7 +192,8 @@ def _should_replace_result( self, result_path: str, result: dict, to_replace_entries: List[BomEntry] ) -> Tuple[bool, str]: """ - Check if a result should be replaced based on the SCANOSS settings + Check if a result should be replaced based on the SCANOSS settings. + Uses priority-based matching: most specific rule wins. Args: result_path (str): Path of the result data @@ -223,60 +205,42 @@ def _should_replace_result( str: The purl to replace with """ result_purls = result.get('purl', []) - for to_replace_entry in to_replace_entries: - to_replace_path = to_replace_entry.get('path') - to_replace_purl = to_replace_entry.get('purl') - to_replace_with = to_replace_entry.get('replace_with') - - if not to_replace_path and not to_replace_purl or not to_replace_with: - continue - if ( - _is_full_match(result_path, result_purls, to_replace_entry) - or (not to_replace_path and to_replace_purl in result_purls) - or (not to_replace_purl and to_replace_path == result_path) - ): - self._print_message(result_path, result_purls, to_replace_entry, 'Replacing') - return True, to_replace_with - + match = find_best_match(result_path, result_purls, to_replace_entries) + if match and match.get('replace_with'): + self._print_message(result_path, result_purls, match, 'Replacing') + return True, match.get('replace_with') return False, None def _should_remove_result(self, result_path: str, result: dict, to_remove_entries: List[BomEntry]) -> bool: """ - Check if a result should be removed based on the SCANOSS settings + Check if a result should be removed based on the SCANOSS settings. + Uses priority-based matching: most specific rule wins. + + Args: + result_path (str): Path of the result data + result (dict): Result to check + to_remove_entries (List[BomEntry]): BOM entries to remove from the result - :param result_path: path of the result data - :param result: result to check - :param to_remove_entries: BOM entries to remove from the result - :return: + Returns: + True if the result should be removed """ result = result[0] if isinstance(result, list) else result result_purls = result.get('purl', []) - - for to_remove_entry in to_remove_entries: - to_remove_path = to_remove_entry.get('path') - to_remove_purl = to_remove_entry.get('purl') - - if not to_remove_path and not to_remove_purl: - continue - if ( - _is_full_match(result_path, result_purls, to_remove_entry) - or (not to_remove_path and to_remove_purl in result_purls) - or (not to_remove_purl and to_remove_path == result_path) - ): - self._print_message(result_path, result_purls, to_remove_entry, 'Removing') - return True - + match = find_best_match(result_path, result_purls, to_remove_entries) + if match: + self._print_message(result_path, result_purls, match, 'Removing') + return True return False def _print_message(self, result_path: str, result_purls: List[str], bom_entry: BomEntry, action: str) -> None: """ Print a message about replacing or removing a result - :param result_path: - :param result_purls: - :param bom_entry: - :param action: - :return: + Args: + result_path (str): Path of the scan result + result_purls (List[str]): Purls from the scan result + bom_entry (BomEntry): Matched BOM entry + action (str): Action being performed """ message = ( f'{_get_match_type_message(result_path, bom_entry, action)} \n' From bc2deafbf8bb175d9a1360f3d4826c552a7b3adb Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 11:52:30 +0100 Subject: [PATCH 427/489] feat(scanner): per-batch SBOM context for path-scoped include/exclude When BOM include/exclude entries have path fields, resolve SBOM context per-batch instead of setting it globally. Each API request now receives only the purls relevant to the files in that batch. - scanossapi: add per-request sbom override parameter to scan() - threadedscanning: pass SBOM alongside WFP through the queue - scanner: track file paths per batch, compute filtered SBOM at flush points, extract paths from WFP for pre-generated fingerprint files Purl-only entries (no path) are always included in every request. When no path-scoped entries exist, the global SBOM behavior is unchanged. --- src/scanoss/scanner.py | 80 ++++++++++++++++++++++++++++----- src/scanoss/scanossapi.py | 10 +++-- src/scanoss/threadedscanning.py | 9 ++-- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 6db4ed40..1fef01c5 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -233,15 +233,54 @@ def __init__( # noqa: PLR0913, PLR0915 self.post_processor = ( ScanPostProcessor(scanoss_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None ) + self._per_batch_sbom = ( + scan_settings is not None and scan_settings.has_path_scoped_bom_entries() + ) self._maybe_set_api_sbom() def _maybe_set_api_sbom(self): if not self.scanoss_settings: return + if self._per_batch_sbom: + self.print_debug('Path-scoped BOM entries detected. SBOM context will be resolved per-batch.') + return sbom = self.scanoss_settings.get_sbom() if sbom: self.scanoss_api.set_sbom(sbom) + def _get_batch_sbom(self, batch_file_paths: List[str]) -> 'dict | None': + """ + Compute SBOM context for a specific batch of files. + Returns None if no per-batch resolution is needed. + + Args: + batch_file_paths: List of file paths in the current batch + Returns: + SBOM payload dict or None + """ + if not self._per_batch_sbom: + return None + return self.scan_settings.get_sbom_for_batch(batch_file_paths) + + @staticmethod + def _extract_file_paths_from_wfp(wfp: str) -> List[str]: + """ + Extract file paths from a WFP string. + WFP file lines have the format: file=,, + + Args: + wfp: WFP string + Returns: + List of file paths + """ + paths = [] + for line in wfp.split('\n'): + if line.startswith(WFP_FILE_START): + parts = line[len(WFP_FILE_START):].split(',', 2) + if len(parts) >= 3: + paths.append(parts[2].strip()) + return paths + @staticmethod def _merge_cli_with_settings(cli_value, settings_value): """Merge CLI value with settings value (two-level priority: settings > cli). @@ -427,6 +466,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 wfp_file_count = 0 # count number of files in each queue post scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified + batch_file_paths = [] # Track file paths for per-batch SBOM resolution to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) for to_scan_file in to_scan_files: @@ -449,20 +489,23 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 # If the WFP is bigger than the max post size and we already have something # stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) + self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) queue_size += 1 scan_block = '' wfp_file_count = 0 + batch_file_paths = [] scan_block += wfp + batch_file_paths.append(to_scan_file) scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # If the scan request block (group of WFPs) is larger than the POST size # or we have reached the file limit, add it to the queue if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) + self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) queue_size += 1 scan_block = '' wfp_file_count = 0 + batch_file_paths = [] if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): @@ -473,7 +516,9 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 success = False # End for loop if self.threaded_scan and scan_block != '': - self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted + self.threaded_scan.queue_add( + scan_block, sbom=self._get_batch_sbom(batch_file_paths) + ) # Make sure all files have been submitted if file_count > 0: if wfp_list is not None: @@ -651,7 +696,7 @@ def scan_file(self, file: str) -> bool: wfp = self.winnowing.wfp_for_file(file, file) if wfp is not None and wfp != '': if self.threaded_scan: - self.threaded_scan.queue_add(wfp) # Submit the WFP for scanning + self.threaded_scan.queue_add(wfp, sbom=self._get_batch_sbom([file])) # Submit the WFP for scanning self.print_debug(f'Scanning {file}...') if self.threaded_scan: success = self.__run_scan_threaded(False, 1) @@ -694,6 +739,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 wfp_file_count = 0 # count number of files in each queue post scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified + batch_file_paths = [] # Track file paths for per-batch SBOM resolution to_scan_files = file_filters.get_filtered_files_from_files(files) for file in to_scan_files: @@ -715,20 +761,23 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 # If the WFP is bigger than the max post size and we already have something # stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) + self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) queue_size += 1 scan_block = '' wfp_file_count = 0 + batch_file_paths = [] scan_block += wfp + batch_file_paths.append(file) scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # If the scan request block (group of WFPs) is larger than the POST size # or we have reached the file limit, add it to the queue if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block) + self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) queue_size += 1 scan_block = '' wfp_file_count = 0 + batch_file_paths = [] if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): @@ -740,7 +789,9 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 # End for loop if self.threaded_scan and scan_block != '': - self.threaded_scan.queue_add(scan_block) # Make sure all files have been submitted + self.threaded_scan.queue_add( + scan_block, sbom=self._get_batch_sbom(batch_file_paths) + ) # Make sure all files have been submitted if file_count > 0: if wfp_list is not None: @@ -798,7 +849,9 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: wfp = self.winnowing.wfp_for_contents(filename, False, contents) if wfp is not None and wfp != '': if self.threaded_scan: - self.threaded_scan.queue_add(wfp) # Submit the WFP for scanning + self.threaded_scan.queue_add( + wfp, sbom=self._get_batch_sbom([filename]) + ) # Submit the WFP for scanning self.print_debug(f'Scanning {filename}...') if self.threaded_scan: success = self.__run_scan_threaded(False, 1) @@ -873,7 +926,9 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 if (wfp_file_count > self.post_file_count or l_size >= self.max_post_size) and wfp: if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') - self.threaded_scan.queue_add(wfp) + self.threaded_scan.queue_add( + wfp, sbom=self._get_batch_sbom(self._extract_file_paths_from_wfp(wfp)) + ) queue_size += 1 wfp = '' wfp_file_count = 0 @@ -888,7 +943,9 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 if scan_block: wfp += scan_block # Store the WFP for the current file if wfp: - self.threaded_scan.queue_add(wfp) + self.threaded_scan.queue_add( + wfp, sbom=self._get_batch_sbom(self._extract_file_paths_from_wfp(wfp)) + ) queue_size += 1 if not self.__run_scan_threaded(scan_started, file_count): @@ -907,7 +964,8 @@ def scan_wfp(self, wfp: str) -> bool: if not wfp: raise Exception('ERROR: Please specify a WFP to scan') raw_output = '{\n' - scan_resp = self.scanoss_api.scan(wfp) + sbom = self._get_batch_sbom(self._extract_file_paths_from_wfp(wfp)) + scan_resp = self.scanoss_api.scan(wfp, sbom=sbom) if scan_resp is not None: for key, value in scan_resp.items(): raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2)) diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 73f0838a..6314d9fe 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -179,19 +179,21 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 if self.proxies: self.session.proxies = self.proxies - def scan(self, wfp: str, context: str = None, scan_id: int = None): # noqa: PLR0912, PLR0915 + def scan(self, wfp: str, context: str = None, scan_id: int = None, sbom: dict = None): # noqa: PLR0912, PLR0915 """ Scan the specified WFP and return the JSON object :param wfp: WFP to scan :param context: Context to help with identification :param scan_id: ID of the scan being run (usually thread id) + :param sbom: Per-request SBOM context (overrides global self.sbom if provided) :return: JSON result object """ request_id = str(uuid.uuid4()) form_data = {} - if self.sbom: - form_data['type'] = self.sbom.get('scan_type') - form_data['assets'] = self.sbom.get('assets') + effective_sbom = sbom if sbom is not None else self.sbom + if effective_sbom: + form_data['type'] = effective_sbom.get('scan_type') + form_data['assets'] = effective_sbom.get('assets') if self.scan_format: form_data['format'] = self.scan_format if self.flags: diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index d0a5cad7..97515104 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -138,15 +138,16 @@ def update_bar(self, amount: int = 0, create: bool = False, file_count: int = 0) except Exception as e: self.print_debug(f'Warning: Update status bar lock failed: {e}. Ignoring.') - def queue_add(self, wfp: str) -> None: + def queue_add(self, wfp: str, sbom: dict = None) -> None: """ Add requests to the queue :param wfp: WFP to add to queue + :param sbom: Per-request SBOM context (optional, overrides global SBOM) """ if wfp is None or wfp == '': self.print_stderr('Warning: empty WFP. Skipping from scan...') else: - self.inputs.put(wfp) + self.inputs.put((wfp, sbom)) def get_queue_size(self) -> int: return self.inputs.qsize() @@ -216,7 +217,7 @@ def worker_post(self) -> None: wfp = None if not self.inputs.empty(): # Only try to get a message if there is one on the queue try: - wfp = self.inputs.get(timeout=5) + wfp, sbom = self.inputs.get(timeout=5) if api_error: # API error encountered, so stop processing anymore requests self.inputs.task_done() # remove request from the queue else: @@ -224,7 +225,7 @@ def worker_post(self) -> None: count = self.__count_files_in_wfp(wfp) if wfp is None or wfp == '': self.print_stderr(f'Warning: Empty WFP in request input: {wfp}') - resp = self.scanapi.scan(wfp, scan_id=current_thread) + resp = self.scanapi.scan(wfp, scan_id=current_thread, sbom=sbom) if resp: self.output.put(resp) # Store the output response to later collection self.update_bar(count) From 440bf8d6829345ceb6e8f90cf389615890d90f93 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 11:52:34 +0100 Subject: [PATCH 428/489] test: add comprehensive tests for folder-level BOM path matching Cover path matching helpers, priority resolution, post-processor folder matching, per-batch SBOM context filtering, and WFP path extraction. 45 new test cases. --- tests/test_bom_path_matching.py | 520 ++++++++++++++++++++++++++++++++ 1 file changed, 520 insertions(+) create mode 100644 tests/test_bom_path_matching.py diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py new file mode 100644 index 00000000..1b19a55c --- /dev/null +++ b/tests/test_bom_path_matching.py @@ -0,0 +1,520 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +import os +import unittest +from pathlib import Path + +from scanoss.scanoss_settings import ( + ScanossSettings, + entry_priority, + find_best_match, + matches_path, +) +from scanoss.scanpostprocessor import ScanPostProcessor + + +class TestMatchesPath(unittest.TestCase): + """Unit tests for the matches_path helper function""" + + def test_empty_entry_path_matches_everything(self): + self.assertTrue(matches_path('', 'src/main.c')) + self.assertTrue(matches_path('', '')) + + def test_exact_file_match(self): + self.assertTrue(matches_path('src/main.c', 'src/main.c')) + + def test_exact_file_no_match(self): + self.assertFalse(matches_path('src/main.c', 'src/other.c')) + + def test_folder_prefix_match(self): + self.assertTrue(matches_path('src/vendor/', 'src/vendor/lib.c')) + self.assertTrue(matches_path('src/vendor/', 'src/vendor/sub/deep.c')) + + def test_folder_no_match(self): + self.assertFalse(matches_path('src/vendor/', 'src/other/lib.c')) + self.assertFalse(matches_path('src/vendor/', 'src/vendorlib.c')) + + def test_folder_root_prefix(self): + self.assertTrue(matches_path('src/', 'src/main.c')) + self.assertTrue(matches_path('src/', 'src/vendor/deep/file.c')) + + def test_exact_path_does_not_prefix_match(self): + """File paths (no trailing slash) should not do prefix matching""" + self.assertFalse(matches_path('src/main.c', 'src/main.cpp')) + + +class TestEntryPriority(unittest.TestCase): + """Unit tests for the entry_priority helper function""" + + def test_path_and_purl(self): + self.assertEqual(entry_priority({'path': 'src/main.c', 'purl': 'pkg:npm/vue'}), 4) + + def test_purl_only(self): + self.assertEqual(entry_priority({'purl': 'pkg:npm/vue'}), 2) + + def test_path_only(self): + self.assertEqual(entry_priority({'path': 'src/vendor/'}), 1) + + def test_empty_entry(self): + self.assertEqual(entry_priority({}), 0) + + def test_empty_strings(self): + self.assertEqual(entry_priority({'path': '', 'purl': ''}), 0) + + +class TestFindBestMatch(unittest.TestCase): + """Unit tests for the find_best_match helper function""" + + def test_no_entries(self): + result = find_best_match('src/main.c', ['pkg:npm/vue'], []) + self.assertIsNone(result) + + def test_purl_only_match(self): + entries = [{'purl': 'pkg:npm/vue'}] + result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) + self.assertEqual(result, entries[0]) + + def test_path_only_match(self): + entries = [{'path': 'src/vendor/'}] + result = find_best_match('src/vendor/lib.c', ['pkg:npm/vue'], entries) + self.assertEqual(result, entries[0]) + + def test_full_match_beats_purl_only(self): + entries = [ + {'purl': 'pkg:npm/vue'}, + {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + ] + result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) + self.assertEqual(result, entries[1]) + + def test_full_match_beats_path_only(self): + entries = [ + {'path': 'src/'}, + {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + ] + result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) + self.assertEqual(result, entries[1]) + + def test_longer_path_wins_on_tie(self): + entries = [ + {'path': 'src/', 'purl': 'pkg:npm/vue'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, + ] + result = find_best_match('src/vendor/lib.c', ['pkg:npm/vue'], entries) + self.assertEqual(result, entries[1]) + + def test_no_match_when_purl_not_in_result(self): + entries = [{'purl': 'pkg:npm/react'}] + result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) + self.assertIsNone(result) + + def test_no_match_when_path_does_not_match(self): + entries = [{'path': 'lib/', 'purl': 'pkg:npm/vue'}] + result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) + self.assertIsNone(result) + + def test_path_only_entry_matches_without_purl(self): + """Path-only remove entries should match regardless of result purls""" + entries = [{'path': 'src/vendor/'}] + result = find_best_match('src/vendor/lib.c', [], entries) + self.assertEqual(result, entries[0]) + + def test_skip_entries_with_no_path_and_no_purl(self): + entries = [{'comment': 'just a comment'}] + result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) + self.assertIsNone(result) + + def test_order_independent(self): + """Best match should be found regardless of entry order""" + entries_a = [ + {'purl': 'pkg:npm/vue'}, + {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + ] + entries_b = [ + {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + {'purl': 'pkg:npm/vue'}, + ] + result_a = find_best_match('src/main.c', ['pkg:npm/vue'], entries_a) + result_b = find_best_match('src/main.c', ['pkg:npm/vue'], entries_b) + self.assertEqual(result_a['path'], 'src/main.c') + self.assertEqual(result_b['path'], 'src/main.c') + + +class TestPostProcessorFolderMatching(unittest.TestCase): + """Test folder-level matching in the post-processor (remove and replace)""" + + def _make_settings(self, settings_data: dict) -> ScanossSettings: + """Create a ScanossSettings instance from a dict without file I/O""" + settings = ScanossSettings() + settings.data = settings_data + return settings + + def test_remove_by_folder_path(self): + """Should remove all results under a folder path""" + settings = self._make_settings({ + 'bom': { + 'remove': [{'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}], + } + }) + results = { + 'src/vendor/lib.c': [{'purl': ['pkg:npm/vue']}], + 'src/vendor/sub/deep.c': [{'purl': ['pkg:npm/vue']}], + 'src/main.c': [{'purl': ['pkg:npm/vue']}], + } + processor = ScanPostProcessor(settings) + processed = processor.load_results(results).post_process() + self.assertNotIn('src/vendor/lib.c', processed) + self.assertNotIn('src/vendor/sub/deep.c', processed) + self.assertIn('src/main.c', processed) + + def test_remove_by_path_only(self): + """Should remove by path only (no purl required)""" + settings = self._make_settings({ + 'bom': { + 'remove': [{'path': 'src/vendor/'}], + } + }) + results = { + 'src/vendor/lib.c': [{'purl': ['pkg:npm/vue']}], + 'src/main.c': [{'purl': ['pkg:npm/react']}], + } + processor = ScanPostProcessor(settings) + processed = processor.load_results(results).post_process() + self.assertNotIn('src/vendor/lib.c', processed) + self.assertIn('src/main.c', processed) + + def test_replace_by_folder_path(self): + """Should replace purls for all results under a folder""" + settings = self._make_settings({ + 'bom': { + 'replace': [{ + 'path': 'src/vendor/', + 'purl': 'pkg:npm/old-lib', + 'replace_with': 'pkg:npm/new-lib', + }], + } + }) + results = { + 'src/vendor/file.c': [{'purl': ['pkg:npm/old-lib']}], + 'src/vendor/sub/deep.c': [{'purl': ['pkg:npm/old-lib']}], + 'src/main.c': [{'purl': ['pkg:npm/old-lib']}], + } + processor = ScanPostProcessor(settings) + processed = processor.load_results(results).post_process() + self.assertEqual(processed['src/vendor/file.c'][0]['purl'], ['pkg:npm/new-lib']) + self.assertEqual(processed['src/vendor/sub/deep.c'][0]['purl'], ['pkg:npm/new-lib']) + self.assertEqual(processed['src/main.c'][0]['purl'], ['pkg:npm/old-lib']) + + def test_priority_specific_file_beats_folder(self): + """A file+purl rule should take priority over a folder+purl rule""" + settings = self._make_settings({ + 'bom': { + 'replace': [ + { + 'path': 'src/vendor/', + 'purl': 'pkg:npm/lib', + 'replace_with': 'pkg:npm/folder-replacement', + }, + { + 'path': 'src/vendor/special.c', + 'purl': 'pkg:npm/lib', + 'replace_with': 'pkg:npm/file-replacement', + }, + ], + } + }) + results = { + 'src/vendor/special.c': [{'purl': ['pkg:npm/lib']}], + 'src/vendor/other.c': [{'purl': ['pkg:npm/lib']}], + } + processor = ScanPostProcessor(settings) + processed = processor.load_results(results).post_process() + # File rule (score 4, longer path) should beat folder rule (score 4, shorter path) + self.assertEqual(processed['src/vendor/special.c'][0]['purl'], ['pkg:npm/file-replacement']) + self.assertEqual(processed['src/vendor/other.c'][0]['purl'], ['pkg:npm/folder-replacement']) + + def test_priority_purl_plus_path_beats_purl_only(self): + """A purl+path rule should take priority over a purl-only rule""" + settings = self._make_settings({ + 'bom': { + 'remove': [ + {'purl': 'pkg:npm/lib'}, # purl-only, score 2 - should NOT match + ], + 'replace': [ + { + 'path': 'src/', + 'purl': 'pkg:npm/lib', + 'replace_with': 'pkg:npm/replacement', + }, + ], + } + }) + # Remove and replace operate independently on results + results = { + 'src/main.c': [{'purl': ['pkg:npm/lib']}], + } + processor = ScanPostProcessor(settings) + processed = processor.load_results(results).post_process() + # The purl-only remove rule matches, so it should be removed + self.assertNotIn('src/main.c', processed) + + def test_deeper_folder_wins(self): + """A deeper folder rule should take priority over a shallower one""" + settings = self._make_settings({ + 'bom': { + 'replace': [ + { + 'path': 'src/', + 'purl': 'pkg:npm/lib', + 'replace_with': 'pkg:npm/shallow-replacement', + }, + { + 'path': 'src/vendor/', + 'purl': 'pkg:npm/lib', + 'replace_with': 'pkg:npm/deep-replacement', + }, + ], + } + }) + results = { + 'src/vendor/file.c': [{'purl': ['pkg:npm/lib']}], + 'src/main.c': [{'purl': ['pkg:npm/lib']}], + } + processor = ScanPostProcessor(settings) + processed = processor.load_results(results).post_process() + self.assertEqual(processed['src/vendor/file.c'][0]['purl'], ['pkg:npm/deep-replacement']) + self.assertEqual(processed['src/main.c'][0]['purl'], ['pkg:npm/shallow-replacement']) + + +class TestSbomForBatch(unittest.TestCase): + """Test per-batch SBOM context resolution""" + + def _make_settings(self, settings_data: dict) -> ScanossSettings: + """Create a ScanossSettings instance from a dict without file I/O""" + settings = ScanossSettings() + settings.data = settings_data + return settings + + def test_purl_only_entries_always_included(self): + """Purl-only include entries should be sent with every batch""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/vue'}, + {'purl': 'pkg:npm/react'}, + ], + } + }) + result = settings.get_sbom_for_batch(['any/file.c']) + self.assertIsNotNone(result) + assets = json.loads(result['assets']) + purls = [c['purl'] for c in assets['components']] + self.assertIn('pkg:npm/vue', purls) + self.assertIn('pkg:npm/react', purls) + self.assertEqual(result['scan_type'], 'identify') + + def test_folder_scoped_entry_included_when_matching(self): + """Folder-scoped entry should be included when batch contains matching files""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, + ], + } + }) + result = settings.get_sbom_for_batch(['src/vendor/lib.c']) + self.assertIsNotNone(result) + assets = json.loads(result['assets']) + purls = [c['purl'] for c in assets['components']] + self.assertIn('pkg:npm/vue', purls) + + def test_folder_scoped_entry_excluded_when_no_match(self): + """Folder-scoped entry should not be included when no batch files match""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, + ], + } + }) + result = settings.get_sbom_for_batch(['lib/other.c']) + self.assertIsNone(result) + + def test_file_scoped_entry_included_when_exact_match(self): + """File-scoped entry should be included when exact file is in batch""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + ], + } + }) + result = settings.get_sbom_for_batch(['src/main.c', 'src/other.c']) + self.assertIsNotNone(result) + assets = json.loads(result['assets']) + purls = [c['purl'] for c in assets['components']] + self.assertIn('pkg:npm/vue', purls) + + def test_file_scoped_entry_excluded_when_no_match(self): + """File-scoped entry should not be included when file is not in batch""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + ], + } + }) + result = settings.get_sbom_for_batch(['src/other.c']) + self.assertIsNone(result) + + def test_mixed_purl_only_and_scoped(self): + """Purl-only entries always included, scoped entries filtered""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/global-lib'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, + {'path': 'lib/', 'purl': 'pkg:npm/lib-only'}, + ], + } + }) + result = settings.get_sbom_for_batch(['src/vendor/file.c']) + self.assertIsNotNone(result) + assets = json.loads(result['assets']) + purls = [c['purl'] for c in assets['components']] + self.assertIn('pkg:npm/global-lib', purls) + self.assertIn('pkg:npm/vendor-lib', purls) + self.assertNotIn('pkg:npm/lib-only', purls) + + def test_exclude_entries(self): + """Exclude entries should use blacklist scan type""" + settings = self._make_settings({ + 'bom': { + 'exclude': [ + {'purl': 'pkg:npm/excluded'}, + ], + } + }) + result = settings.get_sbom_for_batch(['any/file.c']) + self.assertIsNotNone(result) + self.assertEqual(result['scan_type'], 'blacklist') + assets = json.loads(result['assets']) + purls = [c['purl'] for c in assets['components']] + self.assertIn('pkg:npm/excluded', purls) + + def test_no_entries_returns_none(self): + """Should return None when no include or exclude entries exist""" + settings = self._make_settings({ + 'bom': { + 'include': [], + 'exclude': [], + } + }) + result = settings.get_sbom_for_batch(['src/main.c']) + self.assertIsNone(result) + + def test_no_data_returns_none(self): + """Should return None when settings have no data""" + settings = self._make_settings({}) + result = settings.get_sbom_for_batch(['src/main.c']) + self.assertIsNone(result) + + def test_has_path_scoped_bom_entries_true(self): + """Should detect path-scoped include entries""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'src/', 'purl': 'pkg:npm/vue'}, + ], + } + }) + self.assertTrue(settings.has_path_scoped_bom_entries()) + + def test_has_path_scoped_bom_entries_false(self): + """Should return False when no path-scoped entries exist""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/vue'}, + ], + } + }) + self.assertFalse(settings.has_path_scoped_bom_entries()) + + def test_has_path_scoped_bom_entries_exclude(self): + """Should detect path-scoped exclude entries""" + settings = self._make_settings({ + 'bom': { + 'exclude': [ + {'path': 'lib/', 'purl': 'pkg:npm/excluded'}, + ], + } + }) + self.assertTrue(settings.has_path_scoped_bom_entries()) + + def test_deduplicates_purls(self): + """Should not duplicate purls when multiple entries match""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/vue'}, + {'path': 'src/', 'purl': 'pkg:npm/vue'}, + ], + } + }) + result = settings.get_sbom_for_batch(['src/main.c']) + self.assertIsNotNone(result) + assets = json.loads(result['assets']) + purls = [c['purl'] for c in assets['components']] + self.assertEqual(purls.count('pkg:npm/vue'), 1) + + +class TestExtractFilePathsFromWfp(unittest.TestCase): + """Test WFP file path extraction""" + + def test_extract_single_file(self): + from scanoss.scanner import Scanner + wfp = 'file=abc123,1024,src/main.c\n4=abcdef\n' + paths = Scanner._extract_file_paths_from_wfp(wfp) + self.assertEqual(paths, ['src/main.c']) + + def test_extract_multiple_files(self): + from scanoss.scanner import Scanner + wfp = ( + 'file=abc123,1024,src/main.c\n4=abcdef\n' + 'file=def456,2048,src/vendor/lib.c\n4=ghijkl\n' + ) + paths = Scanner._extract_file_paths_from_wfp(wfp) + self.assertEqual(paths, ['src/main.c', 'src/vendor/lib.c']) + + def test_extract_empty_wfp(self): + from scanoss.scanner import Scanner + paths = Scanner._extract_file_paths_from_wfp('') + self.assertEqual(paths, []) + + +if __name__ == '__main__': + unittest.main() From 617b7973ef73a17187909b0cc0d1c143c5926eef Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 12:38:19 +0100 Subject: [PATCH 429/489] refactor(settings): replace BomEntry TypedDict with dataclass hierarchy Introduce a proper class hierarchy for BOM rules mirroring the Java implementation: BomEntry base dataclass with ReplaceRule subclass that adds replace_with and license fields. All get_bom_* methods now convert raw dicts to typed dataclass instances via from_dict() factory methods. --- src/scanoss/scanoss_settings.py | 92 +++++++++++++++++++++----------- src/scanoss/scanpostprocessor.py | 20 +++---- tests/test_bom_path_matching.py | 48 +++++++++-------- 3 files changed, 96 insertions(+), 64 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index caf21ab0..62632140 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -23,6 +23,7 @@ """ import json +from dataclasses import dataclass from pathlib import Path from typing import List, Optional, TypedDict @@ -39,12 +40,35 @@ DEFAULT_SCANOSS_JSON_FILE = Path('scanoss.json') -class BomEntry(TypedDict, total=False): - purl: str - path: str - replace_with: str - comment: str - license: str +@dataclass +class BomEntry: + purl: Optional[str] = None + path: Optional[str] = None + comment: Optional[str] = None + + @classmethod + def from_dict(cls, data: dict) -> 'BomEntry': + return cls( + purl=data.get('purl'), + path=data.get('path'), + comment=data.get('comment'), + ) + + +@dataclass +class ReplaceRule(BomEntry): + replace_with: Optional[str] = None + license: Optional[str] = None + + @classmethod + def from_dict(cls, data: dict) -> 'ReplaceRule': + return cls( + purl=data.get('purl'), + path=data.get('path'), + comment=data.get('comment'), + replace_with=data.get('replace_with'), + license=data.get('license'), + ) def matches_path(entry_path: str, result_path: str) -> bool: @@ -81,8 +105,8 @@ def entry_priority(entry: BomEntry) -> int: Returns: Priority score """ - has_path = bool(entry.get('path')) - has_purl = bool(entry.get('purl')) + has_path = bool(entry.path) + has_purl = bool(entry.purl) if has_path and has_purl: return 4 if has_purl: @@ -110,8 +134,8 @@ def find_best_match(result_path: str, result_purls: List[str], entries: List[Bom best_path_len = -1 for entry in entries: - entry_path = entry.get('path', '') - entry_purl = entry.get('purl', '') + entry_path = entry.path or '' + entry_purl = entry.purl or '' if not entry_path and not entry_purl: continue @@ -279,9 +303,10 @@ def get_bom_include(self) -> List[BomEntry]: list: List of components to include in the scan """ if self.settings_file_type == 'legacy': - return self._get_bom() - return self._get_bom().get('include', []) - + raw = self._get_bom() + else: + raw = self._get_bom().get('include', []) + return [BomEntry.from_dict(entry) for entry in raw] def get_bom_exclude(self) -> List[BomEntry]: """ @@ -290,8 +315,10 @@ def get_bom_exclude(self) -> List[BomEntry]: list: List of components to exclude from the scan """ if self.settings_file_type == 'legacy': - return self._get_bom() - return self._get_bom().get('exclude', []) + raw = self._get_bom() + else: + raw = self._get_bom().get('exclude', []) + return [BomEntry.from_dict(entry) for entry in raw] def get_bom_remove(self) -> List[BomEntry]: """ @@ -300,18 +327,21 @@ def get_bom_remove(self) -> List[BomEntry]: list: List of components to remove from the scan """ if self.settings_file_type == 'legacy': - return self._get_bom() - return self._get_bom().get('remove', []) + raw = self._get_bom() + else: + raw = self._get_bom().get('remove', []) + return [BomEntry.from_dict(entry) for entry in raw] - def get_bom_replace(self) -> List[BomEntry]: + def get_bom_replace(self) -> List[ReplaceRule]: """ Get the list of components to replace in the scan Returns: - list: List of components to replace in the scan + list: List of replace rules """ if self.settings_file_type == 'legacy': return [] - return self._get_bom().get('replace', []) + raw = self._get_bom().get('replace', []) + return [ReplaceRule.from_dict(entry) for entry in raw] def get_sbom(self): """ @@ -372,7 +402,7 @@ def has_path_scoped_bom_entries(self) -> bool: True if any include/exclude entry has a path field """ for entry in self.get_bom_include() + self.get_bom_exclude(): - if entry.get('path'): + if entry.path: return True return False @@ -405,8 +435,8 @@ def get_sbom_for_batch(self, batch_file_paths: List[str]) -> Optional[dict]: filtered_purls = set() for entry in bom_entries: - entry_path = entry.get('path', '') - entry_purl = entry.get('purl', '') + entry_path = entry.path or '' + entry_purl = entry.purl or '' if not entry_purl: continue if not entry_path: @@ -427,32 +457,32 @@ def get_sbom_for_batch(self, batch_file_paths: List[str]) -> Optional[dict]: } @staticmethod - def normalize_bom_entries(bom_entries) -> List[BomEntry]: + def normalize_bom_entries(bom_entries: List[BomEntry]) -> list: """ Normalize the BOM entries by extracting only the purl field. Args: - bom_entries (List[Dict]): List of BOM entries + bom_entries: List of BOM entries Returns: - List: Normalized BOM entries + List of dicts with only the purl field (for API payload) """ normalized_bom_entries = [] for entry in bom_entries: normalized_bom_entries.append( { - 'purl': entry.get('purl', ''), + 'purl': entry.purl or '', } ) return normalized_bom_entries @staticmethod - def _remove_duplicates(bom_entries: List[BomEntry]) -> List[BomEntry]: + def _remove_duplicates(bom_entries: list) -> list: """ - Remove duplicate BOM entries + Remove duplicate BOM entries based on purl field. Args: - bom_entries (List[Dict]): List of BOM entries + bom_entries: List of normalized BOM entry dicts Returns: - List: List of unique BOM entries + List of unique BOM entries """ already_added = set() unique_entries = [] diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 999fb694..c751ddd2 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -27,7 +27,7 @@ from packageurl import PackageURL from packageurl.contrib import purl2url -from .scanoss_settings import BomEntry, ScanossSettings, find_best_match +from .scanoss_settings import BomEntry, ReplaceRule, ScanossSettings, find_best_match from .scanossbase import ScanossBase @@ -43,11 +43,11 @@ def _get_match_type_message(result_path: str, bom_entry: BomEntry, action: str) Returns: str: The message to be printed """ - entry_path = bom_entry.get('path', '') - if entry_path and bom_entry.get('purl'): + entry_path = bom_entry.path or '' + if entry_path and bom_entry.purl: match_kind = 'folder' if entry_path.endswith('/') else 'file' message = f"{action} '{result_path}'. Full match found ({match_kind} + purl)." - elif bom_entry.get('purl'): + elif bom_entry.purl: message = f"{action} '{result_path}'. Found PURL match." else: message = f"{action} '{result_path}'. Found path match." @@ -189,7 +189,7 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di return result def _should_replace_result( - self, result_path: str, result: dict, to_replace_entries: List[BomEntry] + self, result_path: str, result: dict, to_replace_entries: List[ReplaceRule] ) -> Tuple[bool, str]: """ Check if a result should be replaced based on the SCANOSS settings. @@ -198,7 +198,7 @@ def _should_replace_result( Args: result_path (str): Path of the result data result (dict): Result to check - to_replace_entries (List[BomEntry]): BOM entries to replace from the settings file + to_replace_entries (List[ReplaceRule]): Replace rules from the settings file Returns: bool: True if the result should be replaced, False otherwise @@ -206,9 +206,9 @@ def _should_replace_result( """ result_purls = result.get('purl', []) match = find_best_match(result_path, result_purls, to_replace_entries) - if match and match.get('replace_with'): + if match and isinstance(match, ReplaceRule) and match.replace_with: self._print_message(result_path, result_purls, match, 'Replacing') - return True, match.get('replace_with') + return True, match.replace_with return False, None def _should_remove_result(self, result_path: str, result: dict, to_remove_entries: List[BomEntry]) -> bool: @@ -248,8 +248,8 @@ def _print_message(self, result_path: str, result_purls: List[str], bom_entry: B f' - PURLs: {", ".join(result_purls)}\n' f" - Path: '{result_path}'\n" ) - if action == 'Replacing': - message += f" - {action} with '{bom_entry.get('replace_with')}'" + if action == 'Replacing' and isinstance(bom_entry, ReplaceRule): + message += f" - {action} with '{bom_entry.replace_with}'" self.print_debug(message) diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index 1b19a55c..dec0bc4c 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -28,6 +28,8 @@ from pathlib import Path from scanoss.scanoss_settings import ( + BomEntry, + ReplaceRule, ScanossSettings, entry_priority, find_best_match, @@ -70,19 +72,19 @@ class TestEntryPriority(unittest.TestCase): """Unit tests for the entry_priority helper function""" def test_path_and_purl(self): - self.assertEqual(entry_priority({'path': 'src/main.c', 'purl': 'pkg:npm/vue'}), 4) + self.assertEqual(entry_priority(BomEntry(path='src/main.c', purl='pkg:npm/vue')), 4) def test_purl_only(self): - self.assertEqual(entry_priority({'purl': 'pkg:npm/vue'}), 2) + self.assertEqual(entry_priority(BomEntry(purl='pkg:npm/vue')), 2) def test_path_only(self): - self.assertEqual(entry_priority({'path': 'src/vendor/'}), 1) + self.assertEqual(entry_priority(BomEntry(path='src/vendor/')), 1) def test_empty_entry(self): - self.assertEqual(entry_priority({}), 0) + self.assertEqual(entry_priority(BomEntry()), 0) def test_empty_strings(self): - self.assertEqual(entry_priority({'path': '', 'purl': ''}), 0) + self.assertEqual(entry_priority(BomEntry(path='', purl='')), 0) class TestFindBestMatch(unittest.TestCase): @@ -93,74 +95,74 @@ def test_no_entries(self): self.assertIsNone(result) def test_purl_only_match(self): - entries = [{'purl': 'pkg:npm/vue'}] + entries = [BomEntry(purl='pkg:npm/vue')] result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) self.assertEqual(result, entries[0]) def test_path_only_match(self): - entries = [{'path': 'src/vendor/'}] + entries = [BomEntry(path='src/vendor/')] result = find_best_match('src/vendor/lib.c', ['pkg:npm/vue'], entries) self.assertEqual(result, entries[0]) def test_full_match_beats_purl_only(self): entries = [ - {'purl': 'pkg:npm/vue'}, - {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + BomEntry(purl='pkg:npm/vue'), + BomEntry(path='src/main.c', purl='pkg:npm/vue'), ] result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) self.assertEqual(result, entries[1]) def test_full_match_beats_path_only(self): entries = [ - {'path': 'src/'}, - {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + BomEntry(path='src/'), + BomEntry(path='src/main.c', purl='pkg:npm/vue'), ] result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) self.assertEqual(result, entries[1]) def test_longer_path_wins_on_tie(self): entries = [ - {'path': 'src/', 'purl': 'pkg:npm/vue'}, - {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, + BomEntry(path='src/', purl='pkg:npm/vue'), + BomEntry(path='src/vendor/', purl='pkg:npm/vue'), ] result = find_best_match('src/vendor/lib.c', ['pkg:npm/vue'], entries) self.assertEqual(result, entries[1]) def test_no_match_when_purl_not_in_result(self): - entries = [{'purl': 'pkg:npm/react'}] + entries = [BomEntry(purl='pkg:npm/react')] result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) self.assertIsNone(result) def test_no_match_when_path_does_not_match(self): - entries = [{'path': 'lib/', 'purl': 'pkg:npm/vue'}] + entries = [BomEntry(path='lib/', purl='pkg:npm/vue')] result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) self.assertIsNone(result) def test_path_only_entry_matches_without_purl(self): """Path-only remove entries should match regardless of result purls""" - entries = [{'path': 'src/vendor/'}] + entries = [BomEntry(path='src/vendor/')] result = find_best_match('src/vendor/lib.c', [], entries) self.assertEqual(result, entries[0]) def test_skip_entries_with_no_path_and_no_purl(self): - entries = [{'comment': 'just a comment'}] + entries = [BomEntry(comment='just a comment')] result = find_best_match('src/main.c', ['pkg:npm/vue'], entries) self.assertIsNone(result) def test_order_independent(self): """Best match should be found regardless of entry order""" entries_a = [ - {'purl': 'pkg:npm/vue'}, - {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + BomEntry(purl='pkg:npm/vue'), + BomEntry(path='src/main.c', purl='pkg:npm/vue'), ] entries_b = [ - {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, - {'purl': 'pkg:npm/vue'}, + BomEntry(path='src/main.c', purl='pkg:npm/vue'), + BomEntry(purl='pkg:npm/vue'), ] result_a = find_best_match('src/main.c', ['pkg:npm/vue'], entries_a) result_b = find_best_match('src/main.c', ['pkg:npm/vue'], entries_b) - self.assertEqual(result_a['path'], 'src/main.c') - self.assertEqual(result_b['path'], 'src/main.c') + self.assertEqual(result_a.path, 'src/main.c') + self.assertEqual(result_b.path, 'src/main.c') class TestPostProcessorFolderMatching(unittest.TestCase): From 7ffec8bd69f262ff6a5873323b62d32c4e481713 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 15:15:45 +0100 Subject: [PATCH 430/489] test(scanner): add end-to-end tests for SBOM payload in API requests --- tests/test_bom_path_matching.py | 218 ++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index dec0bc4c..d825c0ee 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -24,9 +24,13 @@ import json import os +import shutil +import tempfile import unittest from pathlib import Path +from unittest.mock import MagicMock +from scanoss.scanner import Scanner from scanoss.scanoss_settings import ( BomEntry, ReplaceRule, @@ -518,5 +522,219 @@ def test_extract_empty_wfp(self): self.assertEqual(paths, []) +class TestScannerSbomPayload(unittest.TestCase): + """End-to-end tests: verify Scanner sends the correct SBOM payload in HTTP POST requests""" + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_dir) + + def _create_files(self, file_paths): + """Create test files in self.test_dir with enough content for WFP generation.""" + for rel_path in file_paths: + abs_path = os.path.join(self.test_dir, rel_path) + os.makedirs(os.path.dirname(abs_path), exist_ok=True) + with open(abs_path, 'w') as f: + f.write('/* generated test content */\n' * 20) + + def _make_settings(self, settings_data): + """Create ScanossSettings from a dict without file I/O.""" + settings = ScanossSettings() + settings.data = settings_data + settings.settings_file_type = 'new' + return settings + + def _create_scanner(self, settings=None): + """Create a Scanner with mocked session.post. + + Returns: + (scanner, mock_post) tuple + """ + scanner = Scanner( + scan_settings=settings, + nb_threads=1, + quiet=True, + scan_options=3, # FILES + SNIPPETS, no deps + ) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.ok = True + mock_response.json.return_value = {} + mock_response.text = '{}' + + mock_post = MagicMock(return_value=mock_response) + scanner.scanoss_api.session.post = mock_post + + return scanner, mock_post + + def _extract_payloads(self, mock_post): + """Extract form_data dicts from all session.post calls.""" + payloads = [] + for call in mock_post.call_args_list: + form_data = call.kwargs.get('data', {}) + payloads.append(form_data) + return payloads + + # -- Global SBOM tests (no path-scoped entries) -- + + def test_global_sbom_include_sent_in_post(self): + """When purl-only include entries exist, every POST should contain + type='identify' and the correct purls in assets.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/vue@2.6.12'}, + {'purl': 'pkg:npm/react@17.0.0'}, + ], + } + }) + self._create_files(['src/main.c', 'src/lib.c']) + scanner, mock_post = self._create_scanner(settings) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + for payload in payloads: + self.assertEqual(payload.get('type'), 'identify') + assets = json.loads(payload.get('assets')) + purls = {c['purl'] for c in assets['components']} + self.assertIn('pkg:npm/vue@2.6.12', purls) + self.assertIn('pkg:npm/react@17.0.0', purls) + + def test_global_sbom_exclude_sent_as_blacklist(self): + """When purl-only exclude entries exist, every POST should contain + type='blacklist' and the correct purls in assets.""" + settings = self._make_settings({ + 'bom': { + 'exclude': [ + {'purl': 'pkg:npm/unwanted@1.0.0'}, + ], + } + }) + self._create_files(['src/main.c']) + scanner, mock_post = self._create_scanner(settings) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + for payload in payloads: + self.assertEqual(payload.get('type'), 'blacklist') + assets = json.loads(payload.get('assets')) + purls = {c['purl'] for c in assets['components']} + self.assertIn('pkg:npm/unwanted@1.0.0', purls) + + def test_no_bom_entries_sends_empty_assets(self): + """When settings have empty BOM lists, POST assets should be an empty list + and type should be None (no scan type resolved).""" + settings = self._make_settings({ + 'bom': { + 'include': [], + 'exclude': [], + } + }) + self._create_files(['src/main.c']) + scanner, mock_post = self._create_scanner(settings) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + for payload in payloads: + self.assertIsNone(payload.get('type')) + assets = json.loads(payload.get('assets')) + self.assertEqual(assets, []) + + # -- Per-batch SBOM tests (path-scoped entries) -- + + def test_per_batch_sbom_path_scoped_include(self): + """Path-scoped include: batch with matching files should include + both global and scoped purls.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/global-lib'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, + ], + } + }) + self._create_files(['src/vendor/lib.c', 'src/main.c']) + scanner, mock_post = self._create_scanner(settings) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + # With both files in one batch, vendor/lib.c triggers the scoped entry + for payload in payloads: + self.assertEqual(payload.get('type'), 'identify') + assets = json.loads(payload.get('assets')) + purls = {c['purl'] for c in assets['components']} + self.assertIn('pkg:npm/global-lib', purls) + self.assertIn('pkg:npm/vendor-lib', purls) + + def test_per_batch_sbom_no_matching_paths(self): + """Path-scoped include with no matching files: POST should have no type/assets.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'vendor/', 'purl': 'pkg:npm/vendor-only'}, + ], + } + }) + # Files are NOT under vendor/ + self._create_files(['src/main.c']) + scanner, mock_post = self._create_scanner(settings) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + for payload in payloads: + self.assertNotIn('type', payload) + self.assertNotIn('assets', payload) + + def test_per_batch_sbom_exclude_path_scoped(self): + """Path-scoped exclude: matching batch should contain type='blacklist'.""" + settings = self._make_settings({ + 'bom': { + 'exclude': [ + {'path': 'src/', 'purl': 'pkg:npm/blocked'}, + ], + } + }) + self._create_files(['src/main.c']) + scanner, mock_post = self._create_scanner(settings) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + for payload in payloads: + self.assertEqual(payload.get('type'), 'blacklist') + assets = json.loads(payload.get('assets')) + purls = {c['purl'] for c in assets['components']} + self.assertIn('pkg:npm/blocked', purls) + + # -- No settings test -- + + def test_no_settings_no_sbom_in_payload(self): + """When Scanner has no scan_settings, POST should have no type/assets.""" + self._create_files(['src/main.c']) + scanner, mock_post = self._create_scanner(settings=None) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + for payload in payloads: + self.assertNotIn('type', payload) + self.assertNotIn('assets', payload) + + if __name__ == '__main__': unittest.main() From 6145e2365d79287cf278d4274fc0be9679756481 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 15:28:48 +0100 Subject: [PATCH 431/489] refactor(scanner): simplify SBOM handling by removing path-scoped logic --- src/scanoss/scanner.py | 7 ++-- src/scanoss/scanoss_settings.py | 14 -------- tests/test_bom_path_matching.py | 59 +++++++-------------------------- 3 files changed, 16 insertions(+), 64 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 1fef01c5..1492c5b8 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -251,16 +251,17 @@ def _maybe_set_api_sbom(self): def _get_batch_sbom(self, batch_file_paths: List[str]) -> 'dict | None': """ Compute SBOM context for a specific batch of files. - Returns None if no per-batch resolution is needed. + Purl-only entries are always included; path-scoped entries + are included only when a batch file matches. Args: batch_file_paths: List of file paths in the current batch Returns: SBOM payload dict or None """ - if not self._per_batch_sbom: + if not self.scanoss_settings: return None - return self.scan_settings.get_sbom_for_batch(batch_file_paths) + return self.scanoss_settings.get_sbom_for_batch(batch_file_paths) @staticmethod def _extract_file_paths_from_wfp(wfp: str) -> List[str]: diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 62632140..75c3a895 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -392,20 +392,6 @@ def _get_sbom_assets(self): return self.normalize_bom_entries(self.get_bom_remove()) - def has_path_scoped_bom_entries(self) -> bool: - """ - Check if any include or exclude BOM entries have path-scoped rules. - When path-scoped entries exist, the SBOM context must be resolved per-batch - instead of sent globally with every request. - - Returns: - True if any include/exclude entry has a path field - """ - for entry in self.get_bom_include() + self.get_bom_exclude(): - if entry.path: - return True - return False - def get_sbom_for_batch(self, batch_file_paths: List[str]) -> Optional[dict]: """ Get the SBOM context filtered for a specific batch of files. diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index d825c0ee..b9dc1537 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -448,39 +448,6 @@ def test_no_data_returns_none(self): result = settings.get_sbom_for_batch(['src/main.c']) self.assertIsNone(result) - def test_has_path_scoped_bom_entries_true(self): - """Should detect path-scoped include entries""" - settings = self._make_settings({ - 'bom': { - 'include': [ - {'path': 'src/', 'purl': 'pkg:npm/vue'}, - ], - } - }) - self.assertTrue(settings.has_path_scoped_bom_entries()) - - def test_has_path_scoped_bom_entries_false(self): - """Should return False when no path-scoped entries exist""" - settings = self._make_settings({ - 'bom': { - 'include': [ - {'purl': 'pkg:npm/vue'}, - ], - } - }) - self.assertFalse(settings.has_path_scoped_bom_entries()) - - def test_has_path_scoped_bom_entries_exclude(self): - """Should detect path-scoped exclude entries""" - settings = self._make_settings({ - 'bom': { - 'exclude': [ - {'path': 'lib/', 'purl': 'pkg:npm/excluded'}, - ], - } - }) - self.assertTrue(settings.has_path_scoped_bom_entries()) - def test_deduplicates_purls(self): """Should not duplicate purls when multiple entries match""" settings = self._make_settings({ @@ -578,9 +545,9 @@ def _extract_payloads(self, mock_post): payloads.append(form_data) return payloads - # -- Global SBOM tests (no path-scoped entries) -- + # -- SBOM tests: purl-only entries -- - def test_global_sbom_include_sent_in_post(self): + def test_sbom_include_sent_in_post(self): """When purl-only include entries exist, every POST should contain type='identify' and the correct purls in assets.""" settings = self._make_settings({ @@ -605,7 +572,7 @@ def test_global_sbom_include_sent_in_post(self): self.assertIn('pkg:npm/vue@2.6.12', purls) self.assertIn('pkg:npm/react@17.0.0', purls) - def test_global_sbom_exclude_sent_as_blacklist(self): + def test_sbom_exclude_sent_as_blacklist(self): """When purl-only exclude entries exist, every POST should contain type='blacklist' and the correct purls in assets.""" settings = self._make_settings({ @@ -628,9 +595,8 @@ def test_global_sbom_exclude_sent_as_blacklist(self): purls = {c['purl'] for c in assets['components']} self.assertIn('pkg:npm/unwanted@1.0.0', purls) - def test_no_bom_entries_sends_empty_assets(self): - """When settings have empty BOM lists, POST assets should be an empty list - and type should be None (no scan type resolved).""" + def test_no_bom_entries_no_sbom_in_payload(self): + """When settings have empty BOM lists, POST should have no type/assets.""" settings = self._make_settings({ 'bom': { 'include': [], @@ -645,15 +611,14 @@ def test_no_bom_entries_sends_empty_assets(self): self.assertTrue(mock_post.called, 'Expected at least one POST call') payloads = self._extract_payloads(mock_post) for payload in payloads: - self.assertIsNone(payload.get('type')) - assets = json.loads(payload.get('assets')) - self.assertEqual(assets, []) + self.assertNotIn('type', payload) + self.assertNotIn('assets', payload) - # -- Per-batch SBOM tests (path-scoped entries) -- + # -- SBOM tests: path-scoped entries -- - def test_per_batch_sbom_path_scoped_include(self): + def test_sbom_path_scoped_include(self): """Path-scoped include: batch with matching files should include - both global and scoped purls.""" + both purl-only and scoped purls.""" settings = self._make_settings({ 'bom': { 'include': [ @@ -677,7 +642,7 @@ def test_per_batch_sbom_path_scoped_include(self): self.assertIn('pkg:npm/global-lib', purls) self.assertIn('pkg:npm/vendor-lib', purls) - def test_per_batch_sbom_no_matching_paths(self): + def test_sbom_no_matching_paths(self): """Path-scoped include with no matching files: POST should have no type/assets.""" settings = self._make_settings({ 'bom': { @@ -698,7 +663,7 @@ def test_per_batch_sbom_no_matching_paths(self): self.assertNotIn('type', payload) self.assertNotIn('assets', payload) - def test_per_batch_sbom_exclude_path_scoped(self): + def test_sbom_exclude_path_scoped(self): """Path-scoped exclude: matching batch should contain type='blacklist'.""" settings = self._make_settings({ 'bom': { From b6da0cd6fce0ffcc8bda9245292069f21e475347 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 29 Jan 2026 16:30:32 +0100 Subject: [PATCH 432/489] feat(scanner): flush batch on SBOM context change to isolate path-scoped purls --- src/scanoss/scanner.py | 24 ++++++++ src/scanoss/scanoss_settings.py | 32 ++++++++++ tests/test_bom_path_matching.py | 102 ++++++++++++++++++++++++++++++-- 3 files changed, 152 insertions(+), 6 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 1492c5b8..60d6406e 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -468,6 +468,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified batch_file_paths = [] # Track file paths for per-batch SBOM resolution + batch_purls = None # Track purl context for the current batch to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) for to_scan_file in to_scan_files: @@ -495,8 +496,18 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 scan_block = '' wfp_file_count = 0 batch_file_paths = [] + batch_purls = None + # Flush if the SBOM context changes (different purl set for this file) + file_purls = self.scan_settings.get_matching_purls(to_scan_file) if self.scan_settings else frozenset() + if batch_purls is not None and file_purls != batch_purls and scan_block != '': + self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + batch_file_paths = [] scan_block += wfp batch_file_paths.append(to_scan_file) + batch_purls = file_purls scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # If the scan request block (group of WFPs) is larger than the POST size @@ -507,6 +518,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 scan_block = '' wfp_file_count = 0 batch_file_paths = [] + batch_purls = None if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): @@ -741,6 +753,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified batch_file_paths = [] # Track file paths for per-batch SBOM resolution + batch_purls = None # Track purl context for the current batch to_scan_files = file_filters.get_filtered_files_from_files(files) for file in to_scan_files: @@ -767,8 +780,18 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 scan_block = '' wfp_file_count = 0 batch_file_paths = [] + batch_purls = None + # Flush if the SBOM context changes (different purl set for this file) + file_purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else frozenset() + if batch_purls is not None and file_purls != batch_purls and scan_block != '': + self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + batch_file_paths = [] scan_block += wfp batch_file_paths.append(file) + batch_purls = file_purls scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # If the scan request block (group of WFPs) is larger than the POST size @@ -779,6 +802,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 scan_block = '' wfp_file_count = 0 batch_file_paths = [] + batch_purls = None if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 75c3a895..09e71d4d 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -392,6 +392,38 @@ def _get_sbom_assets(self): return self.normalize_bom_entries(self.get_bom_remove()) + def get_matching_purls(self, file_path: str) -> frozenset: + """ + Get the set of purls that match a specific file path. + Purl-only entries (no path) are always included. + Path-scoped entries are included only if the file matches. + + Args: + file_path: File path to check + + Returns: + frozenset of matching purl strings (empty if no BOM data) + """ + if not self.data: + return frozenset() + + include_entries = self.get_bom_include() + exclude_entries = self.get_bom_exclude() + bom_entries = include_entries or exclude_entries + + if not bom_entries: + return frozenset() + + purls = set() + for entry in bom_entries: + entry_purl = entry.purl or '' + if not entry_purl: + continue + entry_path = entry.path or '' + if not entry_path or matches_path(entry_path, file_path): + purls.add(entry_purl) + return frozenset(purls) + def get_sbom_for_batch(self, batch_file_paths: List[str]) -> Optional[dict]: """ Get the SBOM context filtered for a specific batch of files. diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index b9dc1537..30f5fac6 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -464,6 +464,42 @@ def test_deduplicates_purls(self): purls = [c['purl'] for c in assets['components']] self.assertEqual(purls.count('pkg:npm/vue'), 1) + def test_get_matching_purls_purl_only(self): + """Purl-only entries should match any file path""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/global'}, + {'purl': 'pkg:npm/other'}, + ], + } + }) + result = settings.get_matching_purls('anything/file.c') + self.assertEqual(result, frozenset({'pkg:npm/global', 'pkg:npm/other'})) + + def test_get_matching_purls_folder_scoped(self): + """Folder-scoped entries should only match files under that folder""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/global'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, + ], + } + }) + # File under vendor/ gets both purls + result_vendor = settings.get_matching_purls('src/vendor/lib.c') + self.assertEqual(result_vendor, frozenset({'pkg:npm/global', 'pkg:npm/vendor-lib'})) + # File outside vendor/ gets only global + result_other = settings.get_matching_purls('src/core/main.c') + self.assertEqual(result_other, frozenset({'pkg:npm/global'})) + + def test_get_matching_purls_no_data(self): + """Should return empty frozenset when no data""" + settings = self._make_settings({}) + result = settings.get_matching_purls('src/main.c') + self.assertEqual(result, frozenset()) + class TestExtractFilePathsFromWfp(unittest.TestCase): """Test WFP file path extraction""" @@ -617,8 +653,8 @@ def test_no_bom_entries_no_sbom_in_payload(self): # -- SBOM tests: path-scoped entries -- def test_sbom_path_scoped_include(self): - """Path-scoped include: batch with matching files should include - both purl-only and scoped purls.""" + """Path-scoped include: files with different contexts should be + sent in separate batches with the correct purls each.""" settings = self._make_settings({ 'bom': { 'include': [ @@ -634,13 +670,21 @@ def test_sbom_path_scoped_include(self): self.assertTrue(mock_post.called, 'Expected at least one POST call') payloads = self._extract_payloads(mock_post) - # With both files in one batch, vendor/lib.c triggers the scoped entry + # Context-change flush splits into 2 batches: + # - vendor batch: {global-lib, vendor-lib} + # - non-vendor batch: {global-lib} only + self.assertEqual(len(payloads), 2, + f'Expected 2 POST calls (different contexts), got {len(payloads)}') + + purl_sets = [] for payload in payloads: self.assertEqual(payload.get('type'), 'identify') assets = json.loads(payload.get('assets')) - purls = {c['purl'] for c in assets['components']} - self.assertIn('pkg:npm/global-lib', purls) - self.assertIn('pkg:npm/vendor-lib', purls) + purls = frozenset(c['purl'] for c in assets['components']) + purl_sets.append(purls) + + self.assertIn(frozenset({'pkg:npm/global-lib', 'pkg:npm/vendor-lib'}), purl_sets) + self.assertIn(frozenset({'pkg:npm/global-lib'}), purl_sets) def test_sbom_no_matching_paths(self): """Path-scoped include with no matching files: POST should have no type/assets.""" @@ -685,6 +729,52 @@ def test_sbom_exclude_path_scoped(self): purls = {c['purl'] for c in assets['components']} self.assertIn('pkg:npm/blocked', purls) + # -- Context-change flush tests -- + + def test_context_change_flushes_batch(self): + """Files in folders with different path-scoped purls should be + sent in separate POST requests with only their matching purls.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/global-lib'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, + {'path': 'src/core/', 'purl': 'pkg:npm/core-lib'}, + ], + } + }) + self._create_files([ + 'src/vendor/a.c', + 'src/core/b.c', + ]) + scanner, mock_post = self._create_scanner(settings) + + scanner.scan_folder_with_options(self.test_dir) + + self.assertTrue(mock_post.called, 'Expected at least one POST call') + payloads = self._extract_payloads(mock_post) + + # Should have exactly 2 POST requests (one per folder context) + self.assertEqual(len(payloads), 2, + f'Expected exactly 2 POST calls (one per folder context), got {len(payloads)}') + + # Collect the purl sets from each payload + purl_sets = [] + for payload in payloads: + self.assertEqual(payload.get('type'), 'identify') + assets = json.loads(payload.get('assets')) + purls = frozenset(c['purl'] for c in assets['components']) + purl_sets.append(purls) + + # One payload should have {global, vendor}, the other {global, core} + expected_vendor = frozenset({'pkg:npm/global-lib', 'pkg:npm/vendor-lib'}) + expected_core = frozenset({'pkg:npm/global-lib', 'pkg:npm/core-lib'}) + + self.assertIn(expected_vendor, purl_sets, + f'Expected vendor payload with {expected_vendor}, got {purl_sets}') + self.assertIn(expected_core, purl_sets, + f'Expected core payload with {expected_core}, got {purl_sets}') + # -- No settings test -- def test_no_settings_no_sbom_in_payload(self): From c879f89df2c62a5eff3468667d6cf413b2defc58 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 18:28:51 +0100 Subject: [PATCH 433/489] refactor(scanner): simplify SBOM handling by removing batch file tracking and path-scoped logic --- src/scanoss/scanner.py | 99 +++++++++------ src/scanoss/scanoss_settings.py | 63 ++++------ tests/test_bom_path_matching.py | 205 +++++++++++++++++--------------- 3 files changed, 189 insertions(+), 178 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 60d6406e..77f53267 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -467,8 +467,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 wfp_file_count = 0 # count number of files in each queue post scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified - batch_file_paths = [] # Track file paths for per-batch SBOM resolution - batch_purls = None # Track purl context for the current batch + batch_purls = None # Track purl context for the current batch (ordered list) to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) for to_scan_file in to_scan_files: @@ -491,33 +490,32 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 # If the WFP is bigger than the max post size and we already have something # stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_file_paths = [] batch_purls = None - # Flush if the SBOM context changes (different purl set for this file) - file_purls = self.scan_settings.get_matching_purls(to_scan_file) if self.scan_settings else frozenset() + # Flush if the SBOM context changes (different purl list for this file) + file_purls = self.scan_settings.get_matching_purls(to_scan_file) if self.scan_settings else [] if batch_purls is not None and file_purls != batch_purls and scan_block != '': - self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_file_paths = [] scan_block += wfp - batch_file_paths.append(to_scan_file) batch_purls = file_purls scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # If the scan request block (group of WFPs) is larger than the POST size # or we have reached the file limit, add it to the queue if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_file_paths = [] batch_purls = None if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True @@ -529,9 +527,8 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 success = False # End for loop if self.threaded_scan and scan_block != '': - self.threaded_scan.queue_add( - scan_block, sbom=self._get_batch_sbom(batch_file_paths) - ) # Make sure all files have been submitted + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) # Make sure all files have been submitted if file_count > 0: if wfp_list is not None: @@ -709,7 +706,9 @@ def scan_file(self, file: str) -> bool: wfp = self.winnowing.wfp_for_file(file, file) if wfp is not None and wfp != '': if self.threaded_scan: - self.threaded_scan.queue_add(wfp, sbom=self._get_batch_sbom([file])) # Submit the WFP for scanning + purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else [] + sbom = self.scan_settings.build_sbom_payload(purls) if self.scan_settings else None + self.threaded_scan.queue_add(wfp, sbom=sbom) # Submit the WFP for scanning self.print_debug(f'Scanning {file}...') if self.threaded_scan: success = self.__run_scan_threaded(False, 1) @@ -752,8 +751,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 wfp_file_count = 0 # count number of files in each queue post scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified - batch_file_paths = [] # Track file paths for per-batch SBOM resolution - batch_purls = None # Track purl context for the current batch + batch_purls = None # Track purl context for the current batch (ordered list) to_scan_files = file_filters.get_filtered_files_from_files(files) for file in to_scan_files: @@ -775,33 +773,32 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 # If the WFP is bigger than the max post size and we already have something # stored in the scan block, add it to the queue if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_file_paths = [] batch_purls = None - # Flush if the SBOM context changes (different purl set for this file) - file_purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else frozenset() + # Flush if the SBOM context changes (different purl list for this file) + file_purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else [] if batch_purls is not None and file_purls != batch_purls and scan_block != '': - self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_file_paths = [] scan_block += wfp - batch_file_paths.append(file) batch_purls = file_purls scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # If the scan request block (group of WFPs) is larger than the POST size # or we have reached the file limit, add it to the queue if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - self.threaded_scan.queue_add(scan_block, sbom=self._get_batch_sbom(batch_file_paths)) + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_file_paths = [] batch_purls = None if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True @@ -814,9 +811,8 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 # End for loop if self.threaded_scan and scan_block != '': - self.threaded_scan.queue_add( - scan_block, sbom=self._get_batch_sbom(batch_file_paths) - ) # Make sure all files have been submitted + sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) # Make sure all files have been submitted if file_count > 0: if wfp_list is not None: @@ -874,9 +870,9 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: wfp = self.winnowing.wfp_for_contents(filename, False, contents) if wfp is not None and wfp != '': if self.threaded_scan: - self.threaded_scan.queue_add( - wfp, sbom=self._get_batch_sbom([filename]) - ) # Submit the WFP for scanning + purls = self.scan_settings.get_matching_purls(filename) if self.scan_settings else [] + sbom = self.scan_settings.build_sbom_payload(purls) if self.scan_settings else None + self.threaded_scan.queue_add(wfp, sbom=sbom) # Submit the WFP for scanning self.print_debug(f'Scanning {filename}...') if self.threaded_scan: success = self.__run_scan_threaded(False, 1) @@ -951,9 +947,17 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 if (wfp_file_count > self.post_file_count or l_size >= self.max_post_size) and wfp: if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') - self.threaded_scan.queue_add( - wfp, sbom=self._get_batch_sbom(self._extract_file_paths_from_wfp(wfp)) - ) + file_paths = self._extract_file_paths_from_wfp(wfp) + all_purls = [] + if self.scan_settings and file_paths: + seen = set() + for fp in file_paths: + for p in self.scan_settings.get_matching_purls(fp): + if p not in seen: + seen.add(p) + all_purls.append(p) + sbom = self.scan_settings.build_sbom_payload(all_purls) if self.scan_settings else None + self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 wfp = '' wfp_file_count = 0 @@ -968,9 +972,17 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 if scan_block: wfp += scan_block # Store the WFP for the current file if wfp: - self.threaded_scan.queue_add( - wfp, sbom=self._get_batch_sbom(self._extract_file_paths_from_wfp(wfp)) - ) + file_paths = self._extract_file_paths_from_wfp(wfp) + all_purls = [] + if self.scan_settings and file_paths: + seen = set() + for fp in file_paths: + for p in self.scan_settings.get_matching_purls(fp): + if p not in seen: + seen.add(p) + all_purls.append(p) + sbom = self.scan_settings.build_sbom_payload(all_purls) if self.scan_settings else None + self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 if not self.__run_scan_threaded(scan_started, file_count): @@ -989,7 +1001,16 @@ def scan_wfp(self, wfp: str) -> bool: if not wfp: raise Exception('ERROR: Please specify a WFP to scan') raw_output = '{\n' - sbom = self._get_batch_sbom(self._extract_file_paths_from_wfp(wfp)) + file_paths = self._extract_file_paths_from_wfp(wfp) + all_purls = [] + if self.scan_settings and file_paths: + seen = set() + for fp in file_paths: + for p in self.scan_settings.get_matching_purls(fp): + if p not in seen: + seen.add(p) + all_purls.append(p) + sbom = self.scan_settings.build_sbom_payload(all_purls) if self.scan_settings else None scan_resp = self.scanoss_api.scan(wfp, sbom=sbom) if scan_resp is not None: for key, value in scan_resp.items(): diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 09e71d4d..42f1cada 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -392,83 +392,62 @@ def _get_sbom_assets(self): return self.normalize_bom_entries(self.get_bom_remove()) - def get_matching_purls(self, file_path: str) -> frozenset: + def get_matching_purls(self, file_path: str) -> list: """ - Get the set of purls that match a specific file path. + Get purls matching a file path, ordered by specificity (most specific first). + Purl-only entries (no path) are always included. Path-scoped entries are included only if the file matches. + When multiple rules match the same purl, the highest specificity is used. Args: file_path: File path to check Returns: - frozenset of matching purl strings (empty if no BOM data) + List of purl strings ordered by specificity (most specific first) """ if not self.data: - return frozenset() + return [] include_entries = self.get_bom_include() exclude_entries = self.get_bom_exclude() bom_entries = include_entries or exclude_entries if not bom_entries: - return frozenset() + return [] - purls = set() + purl_scores = {} for entry in bom_entries: entry_purl = entry.purl or '' if not entry_purl: continue entry_path = entry.path or '' if not entry_path or matches_path(entry_path, file_path): - purls.add(entry_purl) - return frozenset(purls) + # Score: priority (0-4) + path length (longer = more specific) + score = entry_priority(entry) + len(entry_path) + if entry_purl not in purl_scores or score > purl_scores[entry_purl]: + purl_scores[entry_purl] = score - def get_sbom_for_batch(self, batch_file_paths: List[str]) -> Optional[dict]: - """ - Get the SBOM context filtered for a specific batch of files. - Only includes purls from entries whose path matches files in the batch. + # Sort by score descending (most specific first) + return sorted(purl_scores.keys(), key=lambda p: -purl_scores[p]) - Purl-only entries (no path) are always included. - File entries are included only if the exact file is in the batch. - Folder entries are included only if any file in the batch is under that folder. + def build_sbom_payload(self, purls: list) -> Optional[dict]: + """ + Build the SBOM payload dict from an ordered list of purls. Args: - batch_file_paths: List of file paths in the current WFP batch + purls: List of purl strings (order is preserved) Returns: - SBOM payload dict with 'assets' and 'scan_type', or None + SBOM payload dict with 'assets' and 'scan_type', or None if no purls """ - if not self.data: + if not purls: return None include_entries = self.get_bom_include() - exclude_entries = self.get_bom_exclude() - - if not include_entries and not exclude_entries: - return None - - bom_entries = include_entries or exclude_entries scan_type = 'identify' if include_entries else 'blacklist' - filtered_purls = set() - for entry in bom_entries: - entry_path = entry.path or '' - entry_purl = entry.purl or '' - if not entry_purl: - continue - if not entry_path: - filtered_purls.add(entry_purl) - continue - for file_path in batch_file_paths: - if matches_path(entry_path, file_path): - filtered_purls.add(entry_purl) - break - - if not filtered_purls: - return None - - components = [{'purl': p} for p in sorted(filtered_purls)] + components = [{'purl': p} for p in purls] # preserve order return { 'assets': json.dumps({'components': components}), 'scan_type': scan_type, diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index 30f5fac6..210d0dc4 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -316,7 +316,7 @@ def test_deeper_folder_wins(self): class TestSbomForBatch(unittest.TestCase): - """Test per-batch SBOM context resolution""" + """Test per-file SBOM context resolution and payload building""" def _make_settings(self, settings_data: dict) -> ScanossSettings: """Create a ScanossSettings instance from a dict without file I/O""" @@ -324,181 +324,192 @@ def _make_settings(self, settings_data: dict) -> ScanossSettings: settings.data = settings_data return settings - def test_purl_only_entries_always_included(self): - """Purl-only include entries should be sent with every batch""" + # -- get_matching_purls tests -- + + def test_get_matching_purls_purl_only(self): + """Purl-only entries should match any file path""" settings = self._make_settings({ 'bom': { 'include': [ - {'purl': 'pkg:npm/vue'}, - {'purl': 'pkg:npm/react'}, + {'purl': 'pkg:npm/global'}, + {'purl': 'pkg:npm/other'}, ], } }) - result = settings.get_sbom_for_batch(['any/file.c']) - self.assertIsNotNone(result) - assets = json.loads(result['assets']) - purls = [c['purl'] for c in assets['components']] - self.assertIn('pkg:npm/vue', purls) - self.assertIn('pkg:npm/react', purls) - self.assertEqual(result['scan_type'], 'identify') + result = settings.get_matching_purls('anything/file.c') + self.assertIsInstance(result, list) + self.assertEqual(set(result), {'pkg:npm/global', 'pkg:npm/other'}) - def test_folder_scoped_entry_included_when_matching(self): - """Folder-scoped entry should be included when batch contains matching files""" + def test_get_matching_purls_folder_scoped(self): + """Folder-scoped entries should only match files under that folder""" settings = self._make_settings({ 'bom': { 'include': [ - {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, + {'purl': 'pkg:npm/global'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, ], } }) - result = settings.get_sbom_for_batch(['src/vendor/lib.c']) - self.assertIsNotNone(result) - assets = json.loads(result['assets']) - purls = [c['purl'] for c in assets['components']] - self.assertIn('pkg:npm/vue', purls) + # File under vendor/ gets both purls + result_vendor = settings.get_matching_purls('src/vendor/lib.c') + self.assertEqual(set(result_vendor), {'pkg:npm/global', 'pkg:npm/vendor-lib'}) + # File outside vendor/ gets only global + result_other = settings.get_matching_purls('src/core/main.c') + self.assertEqual(result_other, ['pkg:npm/global']) - def test_folder_scoped_entry_excluded_when_no_match(self): - """Folder-scoped entry should not be included when no batch files match""" + def test_get_matching_purls_no_data(self): + """Should return empty list when no data""" + settings = self._make_settings({}) + result = settings.get_matching_purls('src/main.c') + self.assertEqual(result, []) + + def test_get_matching_purls_no_entries(self): + """Should return empty list when no include/exclude entries""" settings = self._make_settings({ 'bom': { - 'include': [ - {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, - ], + 'include': [], + 'exclude': [], } }) - result = settings.get_sbom_for_batch(['lib/other.c']) - self.assertIsNone(result) + result = settings.get_matching_purls('src/main.c') + self.assertEqual(result, []) - def test_file_scoped_entry_included_when_exact_match(self): - """File-scoped entry should be included when exact file is in batch""" + def test_get_matching_purls_deduplicates(self): + """Should not duplicate purls when multiple entries match same purl""" settings = self._make_settings({ 'bom': { 'include': [ - {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + {'purl': 'pkg:npm/vue'}, + {'path': 'src/', 'purl': 'pkg:npm/vue'}, ], } }) - result = settings.get_sbom_for_batch(['src/main.c', 'src/other.c']) - self.assertIsNotNone(result) - assets = json.loads(result['assets']) - purls = [c['purl'] for c in assets['components']] - self.assertIn('pkg:npm/vue', purls) + result = settings.get_matching_purls('src/main.c') + self.assertEqual(result.count('pkg:npm/vue'), 1) - def test_file_scoped_entry_excluded_when_no_match(self): - """File-scoped entry should not be included when file is not in batch""" + def test_get_matching_purls_ordered_by_specificity(self): + """Should return purls ordered by specificity (most specific first)""" settings = self._make_settings({ 'bom': { 'include': [ - {'path': 'src/main.c', 'purl': 'pkg:npm/vue'}, + {'purl': 'pkg:npm/global'}, # score: 2 (purl only) + {'path': 'src/', 'purl': 'pkg:npm/src-lib'}, # score: 4 + 4 = 8 + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, # score: 4 + 11 = 15 ], } }) - result = settings.get_sbom_for_batch(['src/other.c']) - self.assertIsNone(result) + result = settings.get_matching_purls('src/vendor/lib.c') + # Most specific first: vendor-lib (15), src-lib (8), global (2) + self.assertEqual(result, ['pkg:npm/vendor-lib', 'pkg:npm/src-lib', 'pkg:npm/global']) - def test_mixed_purl_only_and_scoped(self): - """Purl-only entries always included, scoped entries filtered""" + def test_get_matching_purls_file_path_most_specific(self): + """File path should be more specific than folder path""" settings = self._make_settings({ 'bom': { 'include': [ - {'purl': 'pkg:npm/global-lib'}, - {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, - {'path': 'lib/', 'purl': 'pkg:npm/lib-only'}, + {'path': 'src/', 'purl': 'pkg:npm/folder-lib'}, + {'path': 'src/main.c', 'purl': 'pkg:npm/file-lib'}, ], } }) - result = settings.get_sbom_for_batch(['src/vendor/file.c']) + result = settings.get_matching_purls('src/main.c') + # File path (10 chars) more specific than folder path (4 chars) + self.assertEqual(result[0], 'pkg:npm/file-lib') + + # -- build_sbom_payload tests -- + + def test_build_sbom_payload_identify(self): + """Should return identify scan type for include entries""" + settings = self._make_settings({ + 'bom': { + 'include': [{'purl': 'pkg:npm/vue'}], + } + }) + result = settings.build_sbom_payload(['pkg:npm/vue', 'pkg:npm/react']) self.assertIsNotNone(result) + self.assertEqual(result['scan_type'], 'identify') assets = json.loads(result['assets']) - purls = [c['purl'] for c in assets['components']] - self.assertIn('pkg:npm/global-lib', purls) - self.assertIn('pkg:npm/vendor-lib', purls) - self.assertNotIn('pkg:npm/lib-only', purls) + # Order should be preserved + self.assertEqual(assets['components'], [{'purl': 'pkg:npm/vue'}, {'purl': 'pkg:npm/react'}]) - def test_exclude_entries(self): - """Exclude entries should use blacklist scan type""" + def test_build_sbom_payload_blacklist(self): + """Should return blacklist scan type for exclude entries""" settings = self._make_settings({ 'bom': { - 'exclude': [ - {'purl': 'pkg:npm/excluded'}, - ], + 'exclude': [{'purl': 'pkg:npm/excluded'}], } }) - result = settings.get_sbom_for_batch(['any/file.c']) + result = settings.build_sbom_payload(['pkg:npm/excluded']) self.assertIsNotNone(result) self.assertEqual(result['scan_type'], 'blacklist') - assets = json.loads(result['assets']) - purls = [c['purl'] for c in assets['components']] - self.assertIn('pkg:npm/excluded', purls) - def test_no_entries_returns_none(self): - """Should return None when no include or exclude entries exist""" + def test_build_sbom_payload_empty_purls(self): + """Should return None for empty purls list""" settings = self._make_settings({ 'bom': { - 'include': [], - 'exclude': [], + 'include': [{'purl': 'pkg:npm/vue'}], } }) - result = settings.get_sbom_for_batch(['src/main.c']) + result = settings.build_sbom_payload([]) self.assertIsNone(result) - def test_no_data_returns_none(self): - """Should return None when settings have no data""" - settings = self._make_settings({}) - result = settings.get_sbom_for_batch(['src/main.c']) - self.assertIsNone(result) + def test_build_sbom_payload_preserves_order(self): + """Should preserve the order of purls in the payload""" + settings = self._make_settings({ + 'bom': { + 'include': [{'purl': 'pkg:npm/a'}], + } + }) + purls = ['pkg:npm/c', 'pkg:npm/a', 'pkg:npm/b'] + result = settings.build_sbom_payload(purls) + assets = json.loads(result['assets']) + self.assertEqual([c['purl'] for c in assets['components']], purls) - def test_deduplicates_purls(self): - """Should not duplicate purls when multiple entries match""" + # -- Integration tests (get_matching_purls + build_sbom_payload) -- + + def test_folder_scoped_entry_included_when_matching(self): + """Folder-scoped entry should be included when file matches""" settings = self._make_settings({ 'bom': { 'include': [ - {'purl': 'pkg:npm/vue'}, - {'path': 'src/', 'purl': 'pkg:npm/vue'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, ], } }) - result = settings.get_sbom_for_batch(['src/main.c']) + purls = settings.get_matching_purls('src/vendor/lib.c') + result = settings.build_sbom_payload(purls) self.assertIsNotNone(result) assets = json.loads(result['assets']) - purls = [c['purl'] for c in assets['components']] - self.assertEqual(purls.count('pkg:npm/vue'), 1) + self.assertEqual([c['purl'] for c in assets['components']], ['pkg:npm/vue']) - def test_get_matching_purls_purl_only(self): - """Purl-only entries should match any file path""" + def test_folder_scoped_entry_excluded_when_no_match(self): + """Folder-scoped entry should not be included when file doesn't match""" settings = self._make_settings({ 'bom': { 'include': [ - {'purl': 'pkg:npm/global'}, - {'purl': 'pkg:npm/other'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/vue'}, ], } }) - result = settings.get_matching_purls('anything/file.c') - self.assertEqual(result, frozenset({'pkg:npm/global', 'pkg:npm/other'})) + purls = settings.get_matching_purls('lib/other.c') + result = settings.build_sbom_payload(purls) + self.assertIsNone(result) - def test_get_matching_purls_folder_scoped(self): - """Folder-scoped entries should only match files under that folder""" + def test_mixed_purl_only_and_scoped(self): + """Purl-only entries always included, scoped entries filtered by path""" settings = self._make_settings({ 'bom': { 'include': [ - {'purl': 'pkg:npm/global'}, + {'purl': 'pkg:npm/global-lib'}, {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, + {'path': 'lib/', 'purl': 'pkg:npm/lib-only'}, ], } }) - # File under vendor/ gets both purls - result_vendor = settings.get_matching_purls('src/vendor/lib.c') - self.assertEqual(result_vendor, frozenset({'pkg:npm/global', 'pkg:npm/vendor-lib'})) - # File outside vendor/ gets only global - result_other = settings.get_matching_purls('src/core/main.c') - self.assertEqual(result_other, frozenset({'pkg:npm/global'})) - - def test_get_matching_purls_no_data(self): - """Should return empty frozenset when no data""" - settings = self._make_settings({}) - result = settings.get_matching_purls('src/main.c') - self.assertEqual(result, frozenset()) + purls = settings.get_matching_purls('src/vendor/file.c') + self.assertIn('pkg:npm/global-lib', purls) + self.assertIn('pkg:npm/vendor-lib', purls) + self.assertNotIn('pkg:npm/lib-only', purls) class TestExtractFilePathsFromWfp(unittest.TestCase): From 53337eaffe0c91566fd9ba7e58f899caade5c3ef Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 18:30:42 +0100 Subject: [PATCH 434/489] test(bom): add comprehensive tests for path matching and SBOM ordering logic --- tests/test_bom_path_matching.py | 123 ++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index 210d0dc4..a0d7d3b0 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -801,6 +801,129 @@ def test_no_settings_no_sbom_in_payload(self): self.assertNotIn('type', payload) self.assertNotIn('assets', payload) + # -- Corner case tests: priority ordering -- + + def test_payload_preserves_specificity_order(self): + """Purls in payload should be ordered by specificity (most specific first).""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/global'}, # score: 2 + {'path': 'src/', 'purl': 'pkg:npm/src-lib'}, # score: 4 + 4 = 8 + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor'}, # score: 4 + 11 = 15 + ], + } + }) + self._create_files(['src/vendor/lib.c']) + scanner, mock_post = self._create_scanner(settings) + scanner.scan_folder_with_options(self.test_dir) + + payloads = self._extract_payloads(mock_post) + self.assertEqual(len(payloads), 1) + assets = json.loads(payloads[0]['assets']) + purl_order = [c['purl'] for c in assets['components']] + # Most specific first + self.assertEqual(purl_order, ['pkg:npm/vendor', 'pkg:npm/src-lib', 'pkg:npm/global']) + + def test_same_context_batches_together(self): + """Files with identical purl context should batch into single POST.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/global'}, + {'path': 'src/', 'purl': 'pkg:npm/src-lib'}, + ], + } + }) + # All files under src/ → same context + self._create_files(['src/a.c', 'src/b.c', 'src/c.c']) + scanner, mock_post = self._create_scanner(settings) + scanner.scan_folder_with_options(self.test_dir) + + payloads = self._extract_payloads(mock_post) + self.assertEqual(len(payloads), 1, 'Expected single POST for same context') + + def test_nested_folder_deeper_wins(self): + """Deeper folder rule should produce higher-priority purl in payload.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'src/', 'purl': 'pkg:npm/shallow'}, + {'path': 'src/vendor/', 'purl': 'pkg:npm/deep'}, + ], + } + }) + self._create_files(['src/vendor/lib.c']) + scanner, mock_post = self._create_scanner(settings) + scanner.scan_folder_with_options(self.test_dir) + + payloads = self._extract_payloads(mock_post) + self.assertEqual(len(payloads), 1) + assets = json.loads(payloads[0]['assets']) + purl_order = [c['purl'] for c in assets['components']] + # deep (score 15) before shallow (score 8) + self.assertEqual(purl_order[0], 'pkg:npm/deep') + + def test_file_path_beats_folder_path_ordering(self): + """File-specific rule should appear before folder rule for ordering.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'src/', 'purl': 'pkg:npm/folder-lib'}, # score: 8 + {'path': 'src/main.c', 'purl': 'pkg:npm/file-lib'}, # score: 14 + ], + } + }) + self._create_files(['src/main.c']) + scanner, mock_post = self._create_scanner(settings) + scanner.scan_folder_with_options(self.test_dir) + + payloads = self._extract_payloads(mock_post) + self.assertEqual(len(payloads), 1) + assets = json.loads(payloads[0]['assets']) + purl_order = [c['purl'] for c in assets['components']] + self.assertEqual(purl_order[0], 'pkg:npm/file-lib') + + def test_three_contexts_three_posts(self): + """Files in 3 different folder contexts should result in 3 POSTs.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'vendor/', 'purl': 'pkg:npm/vendor'}, + {'path': 'core/', 'purl': 'pkg:npm/core'}, + {'path': 'lib/', 'purl': 'pkg:npm/lib'}, + ], + } + }) + self._create_files(['vendor/a.c', 'core/b.c', 'lib/c.c']) + scanner, mock_post = self._create_scanner(settings) + scanner.scan_folder_with_options(self.test_dir) + + payloads = self._extract_payloads(mock_post) + self.assertEqual(len(payloads), 3) + + def test_mixed_matching_and_non_matching(self): + """Files outside all path rules should have no SBOM, inside should have SBOM.""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'vendor/', 'purl': 'pkg:npm/vendor-lib'}, + ], + } + }) + self._create_files(['vendor/lib.c', 'src/main.c']) + scanner, mock_post = self._create_scanner(settings) + scanner.scan_folder_with_options(self.test_dir) + + payloads = self._extract_payloads(mock_post) + # 2 batches: one with SBOM (vendor/), one without (src/) + self.assertEqual(len(payloads), 2) + + has_sbom = [p for p in payloads if 'type' in p] + no_sbom = [p for p in payloads if 'type' not in p] + self.assertEqual(len(has_sbom), 1) + self.assertEqual(len(no_sbom), 1) + if __name__ == '__main__': unittest.main() From 06921e0e5621afe0fac7eea410eb23d8eca0d413 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 18:57:55 +0100 Subject: [PATCH 435/489] refactor(bom): consolidate path matching and priority logic into BomEntry class --- src/scanoss/scanoss_settings.py | 92 +++++++++++++++------------------ tests/test_bom_path_matching.py | 39 +++++++------- 2 files changed, 62 insertions(+), 69 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 42f1cada..95b3e12b 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -54,6 +54,45 @@ def from_dict(cls, data: dict) -> 'BomEntry': comment=data.get('comment'), ) + def matches_path(self, result_path: str) -> bool: + """ + Check if this entry's path matches a result path. + Folder paths (ending with '/') use prefix matching; file paths use exact matching. + + Args: + result_path: Path from the scan result + + Returns: + True if this entry's path matches the result path + """ + if not self.path: + return True + if self.path.endswith('/'): + return result_path.startswith(self.path) + return self.path == result_path + + @property + def priority(self) -> int: + """ + Priority score for this BOM entry. Higher score means higher priority (more specific). + + Score 4: both path and purl (most specific) + Score 2: purl only + Score 1: path only (remove only, no purl) + + Returns: + Priority score + """ + has_path = bool(self.path) + has_purl = bool(self.purl) + if has_path and has_purl: + return 4 + if has_purl: + return 2 + if has_path: + return 1 + return 0 + @dataclass class ReplaceRule(BomEntry): @@ -71,51 +110,6 @@ def from_dict(cls, data: dict) -> 'ReplaceRule': ) -def matches_path(entry_path: str, result_path: str) -> bool: - """ - Check if a BOM entry path matches a result path. - Folder paths (ending with '/') use prefix matching; file paths use exact matching. - - Args: - entry_path: Path from the BOM entry - result_path: Path from the scan result - - Returns: - True if the entry path matches the result path - """ - if not entry_path: - return True - if entry_path.endswith('/'): - return result_path.startswith(entry_path) - return entry_path == result_path - - -def entry_priority(entry: BomEntry) -> int: - """ - Calculate the priority score for a BOM entry. - Higher score means higher priority (more specific). - - Score 4: both path and purl (most specific) - Score 2: purl only - Score 1: path only (remove only, no purl) - - Args: - entry: BOM entry to evaluate - - Returns: - Priority score - """ - has_path = bool(entry.path) - has_purl = bool(entry.purl) - if has_path and has_purl: - return 4 - if has_purl: - return 2 - if has_path: - return 1 - return 0 - - def find_best_match(result_path: str, result_purls: List[str], entries: List[BomEntry]) -> Optional[BomEntry]: """ Find the highest-priority BOM entry that matches a result. @@ -139,12 +133,12 @@ def find_best_match(result_path: str, result_purls: List[str], entries: List[Bom if not entry_path and not entry_purl: continue - if entry_path and not matches_path(entry_path, result_path): + if entry_path and not entry.matches_path(result_path): continue if entry_purl and (not result_purls or entry_purl not in result_purls): continue - score = entry_priority(entry) + score = entry.priority path_len = len(entry_path) if score > best_score or (score == best_score and path_len > best_path_len): best_entry = entry @@ -422,9 +416,9 @@ def get_matching_purls(self, file_path: str) -> list: if not entry_purl: continue entry_path = entry.path or '' - if not entry_path or matches_path(entry_path, file_path): + if not entry_path or entry.matches_path(file_path): # Score: priority (0-4) + path length (longer = more specific) - score = entry_priority(entry) + len(entry_path) + score = entry.priority + len(entry_path) if entry_purl not in purl_scores or score > purl_scores[entry_purl]: purl_scores[entry_purl] = score diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index a0d7d3b0..dc09595c 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -35,60 +35,59 @@ BomEntry, ReplaceRule, ScanossSettings, - entry_priority, find_best_match, - matches_path, ) from scanoss.scanpostprocessor import ScanPostProcessor class TestMatchesPath(unittest.TestCase): - """Unit tests for the matches_path helper function""" + """Unit tests for the BomEntry.matches_path method""" def test_empty_entry_path_matches_everything(self): - self.assertTrue(matches_path('', 'src/main.c')) - self.assertTrue(matches_path('', '')) + self.assertTrue(BomEntry(path='').matches_path('src/main.c')) + self.assertTrue(BomEntry(path=None).matches_path('src/main.c')) + self.assertTrue(BomEntry(path='').matches_path('')) def test_exact_file_match(self): - self.assertTrue(matches_path('src/main.c', 'src/main.c')) + self.assertTrue(BomEntry(path='src/main.c').matches_path('src/main.c')) def test_exact_file_no_match(self): - self.assertFalse(matches_path('src/main.c', 'src/other.c')) + self.assertFalse(BomEntry(path='src/main.c').matches_path('src/other.c')) def test_folder_prefix_match(self): - self.assertTrue(matches_path('src/vendor/', 'src/vendor/lib.c')) - self.assertTrue(matches_path('src/vendor/', 'src/vendor/sub/deep.c')) + self.assertTrue(BomEntry(path='src/vendor/').matches_path('src/vendor/lib.c')) + self.assertTrue(BomEntry(path='src/vendor/').matches_path('src/vendor/sub/deep.c')) def test_folder_no_match(self): - self.assertFalse(matches_path('src/vendor/', 'src/other/lib.c')) - self.assertFalse(matches_path('src/vendor/', 'src/vendorlib.c')) + self.assertFalse(BomEntry(path='src/vendor/').matches_path('src/other/lib.c')) + self.assertFalse(BomEntry(path='src/vendor/').matches_path('src/vendorlib.c')) def test_folder_root_prefix(self): - self.assertTrue(matches_path('src/', 'src/main.c')) - self.assertTrue(matches_path('src/', 'src/vendor/deep/file.c')) + self.assertTrue(BomEntry(path='src/').matches_path('src/main.c')) + self.assertTrue(BomEntry(path='src/').matches_path('src/vendor/deep/file.c')) def test_exact_path_does_not_prefix_match(self): """File paths (no trailing slash) should not do prefix matching""" - self.assertFalse(matches_path('src/main.c', 'src/main.cpp')) + self.assertFalse(BomEntry(path='src/main.c').matches_path('src/main.cpp')) class TestEntryPriority(unittest.TestCase): - """Unit tests for the entry_priority helper function""" + """Unit tests for the BomEntry.priority property""" def test_path_and_purl(self): - self.assertEqual(entry_priority(BomEntry(path='src/main.c', purl='pkg:npm/vue')), 4) + self.assertEqual(BomEntry(path='src/main.c', purl='pkg:npm/vue').priority, 4) def test_purl_only(self): - self.assertEqual(entry_priority(BomEntry(purl='pkg:npm/vue')), 2) + self.assertEqual(BomEntry(purl='pkg:npm/vue').priority, 2) def test_path_only(self): - self.assertEqual(entry_priority(BomEntry(path='src/vendor/')), 1) + self.assertEqual(BomEntry(path='src/vendor/').priority, 1) def test_empty_entry(self): - self.assertEqual(entry_priority(BomEntry()), 0) + self.assertEqual(BomEntry().priority, 0) def test_empty_strings(self): - self.assertEqual(entry_priority(BomEntry(path='', purl='')), 0) + self.assertEqual(BomEntry(path='', purl='').priority, 0) class TestFindBestMatch(unittest.TestCase): From 14e22f21a50c9289fcf37fc82d95350a97768588 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 19:19:57 +0100 Subject: [PATCH 436/489] refactor(scanner): clarify batch flush logic with descriptive comments --- src/scanoss/scanner.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 77f53267..61e3f55a 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -487,8 +487,8 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something - # stored in the scan block, add it to the queue + + # FLUSH #1: Current file won't fit in batch (size limit) if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) @@ -496,7 +496,8 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 scan_block = '' wfp_file_count = 0 batch_purls = None - # Flush if the SBOM context changes (different purl list for this file) + + # FLUSH #2: SBOM context changed (different purl list for this file) file_purls = self.scan_settings.get_matching_purls(to_scan_file) if self.scan_settings else [] if batch_purls is not None and file_purls != batch_purls and scan_block != '': sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None @@ -504,12 +505,14 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 queue_size += 1 scan_block = '' wfp_file_count = 0 + + # ADD current file to batch scan_block += wfp batch_purls = file_purls scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 - # If the scan request block (group of WFPs) is larger than the POST size - # or we have reached the file limit, add it to the queue + + # FLUSH #3: Batch is full (file count or size limit) if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) From 1df0d5b7e1281061cece8d0ff22a118bd36c434d Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 19:26:56 +0100 Subject: [PATCH 437/489] refactor(scanner): streamline batch flushing conditions and improve context clarity with structured comments --- src/scanoss/scanner.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 61e3f55a..ade42200 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -488,8 +488,11 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) - # FLUSH #1: Current file won't fit in batch (size limit) - if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + # Compute SBOM context for this file + file_purls = self.scan_settings.get_matching_purls(to_scan_file) if self.scan_settings else [] + + # FLUSH: Context changed (different purl list for this file) + if scan_block != '' and batch_purls is not None and file_purls != batch_purls: sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 @@ -497,14 +500,14 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 wfp_file_count = 0 batch_purls = None - # FLUSH #2: SBOM context changed (different purl list for this file) - file_purls = self.scan_settings.get_matching_purls(to_scan_file) if self.scan_settings else [] - if batch_purls is not None and file_purls != batch_purls and scan_block != '': + # FLUSH: Current file won't fit in batch (size limit) + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 + batch_purls = None # ADD current file to batch scan_block += wfp @@ -512,7 +515,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 - # FLUSH #3: Batch is full (file count or size limit) + # FLUSH: Batch is full (file count or size limit) if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) @@ -773,29 +776,35 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 file_count += 1 if self.threaded_scan: wfp_size = len(wfp.encode('utf-8')) - # If the WFP is bigger than the max post size and we already have something - # stored in the scan block, add it to the queue - if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + + # Compute SBOM context for this file + file_purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else [] + + # FLUSH: Context changed (different purl list for this file) + if batch_purls is not None and file_purls != batch_purls and scan_block != '': sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 batch_purls = None - # Flush if the SBOM context changes (different purl list for this file) - file_purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else [] - if batch_purls is not None and file_purls != batch_purls and scan_block != '': + + # FLUSH: Current file won't fit in batch (size limit) + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 + batch_purls = None + + # ADD current file to batch scan_block += wfp batch_purls = file_purls scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 - # If the scan request block (group of WFPs) is larger than the POST size - # or we have reached the file limit, add it to the queue + + # FLUSH: Batch is full (file count or size limit) if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None self.threaded_scan.queue_add(scan_block, sbom=sbom) From d6562e505ea8bf9921c3c882fd6f811a67bf09b5 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 22:19:42 +0100 Subject: [PATCH 438/489] refactor(scanner): unify SBOM context handling with SbomContext class and streamline batch flushing logic --- src/scanoss/scanner.py | 91 ++++++-------- src/scanoss/scanoss_settings.py | 88 +++++++++---- tests/test_bom_path_matching.py | 210 +++++++++++++++++++++----------- 3 files changed, 238 insertions(+), 151 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index ade42200..f19c2f8b 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -41,7 +41,7 @@ from .cyclonedx import CycloneDx from .scan_settings_builder import ScanSettingsBuilder from .scancodedeps import ScancodeDeps -from .scanoss_settings import ScanossSettings +from .scanoss_settings import SbomContext, ScanossSettings from .scanossapi import ScanossApi from .scanossbase import ScanossBase from .scanossgrpc import ScanossGrpc @@ -467,7 +467,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 wfp_file_count = 0 # count number of files in each queue post scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified - batch_purls = None # Track purl context for the current batch (ordered list) + batch_context = None # Track SBOM context (purls, scan_type) for the current batch to_scan_files = file_filters.get_filtered_files_from_folder(scan_dir) for to_scan_file in to_scan_files: @@ -489,40 +489,40 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 wfp_size = len(wfp.encode('utf-8')) # Compute SBOM context for this file - file_purls = self.scan_settings.get_matching_purls(to_scan_file) if self.scan_settings else [] + file_context = self.scan_settings.get_sbom_context(to_scan_file) if self.scan_settings else SbomContext.empty() - # FLUSH: Context changed (different purl list for this file) - if scan_block != '' and batch_purls is not None and file_purls != batch_purls: - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + # FLUSH: Context changed (different purls or scan_type) + if scan_block != '' and batch_context is not None and file_context != batch_context: + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_purls = None + batch_context = None # FLUSH: Current file won't fit in batch (size limit) if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_purls = None + batch_context = None # ADD current file to batch scan_block += wfp - batch_purls = file_purls + batch_context = file_context scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # FLUSH: Batch is full (file count or size limit) if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_purls = None + batch_context = None if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): @@ -533,7 +533,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 success = False # End for loop if self.threaded_scan and scan_block != '': - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) # Make sure all files have been submitted if file_count > 0: @@ -712,8 +712,8 @@ def scan_file(self, file: str) -> bool: wfp = self.winnowing.wfp_for_file(file, file) if wfp is not None and wfp != '': if self.threaded_scan: - purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else [] - sbom = self.scan_settings.build_sbom_payload(purls) if self.scan_settings else None + file_context = self.scan_settings.get_sbom_context(file) if self.scan_settings else SbomContext.empty() + sbom = file_context.to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) # Submit the WFP for scanning self.print_debug(f'Scanning {file}...') if self.threaded_scan: @@ -757,7 +757,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 wfp_file_count = 0 # count number of files in each queue post scan_started = False wfp_list = [] if self.wfp_output else None # Collect WFPs if output file is specified - batch_purls = None # Track purl context for the current batch (ordered list) + batch_context = None # Track SBOM context (purls, scan_type) for the current batch to_scan_files = file_filters.get_filtered_files_from_files(files) for file in to_scan_files: @@ -778,40 +778,40 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 wfp_size = len(wfp.encode('utf-8')) # Compute SBOM context for this file - file_purls = self.scan_settings.get_matching_purls(file) if self.scan_settings else [] + file_context = self.scan_settings.get_sbom_context(file) if self.scan_settings else SbomContext.empty() - # FLUSH: Context changed (different purl list for this file) - if batch_purls is not None and file_purls != batch_purls and scan_block != '': - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + # FLUSH: Context changed (different purls or scan_type) + if batch_context is not None and file_context != batch_context and scan_block != '': + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_purls = None + batch_context = None # FLUSH: Current file won't fit in batch (size limit) if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_purls = None + batch_context = None # ADD current file to batch scan_block += wfp - batch_purls = file_purls + batch_context = file_context scan_size = len(scan_block.encode('utf-8')) wfp_file_count += 1 # FLUSH: Batch is full (file count or size limit) if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 scan_block = '' wfp_file_count = 0 - batch_purls = None + batch_context = None if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): @@ -823,7 +823,7 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 # End for loop if self.threaded_scan and scan_block != '': - sbom = self.scan_settings.build_sbom_payload(batch_purls) if self.scan_settings else None + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(scan_block, sbom=sbom) # Make sure all files have been submitted if file_count > 0: @@ -882,8 +882,8 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: wfp = self.winnowing.wfp_for_contents(filename, False, contents) if wfp is not None and wfp != '': if self.threaded_scan: - purls = self.scan_settings.get_matching_purls(filename) if self.scan_settings else [] - sbom = self.scan_settings.build_sbom_payload(purls) if self.scan_settings else None + file_context = self.scan_settings.get_sbom_context(filename) if self.scan_settings else SbomContext.empty() + sbom = file_context.to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) # Submit the WFP for scanning self.print_debug(f'Scanning {filename}...') if self.threaded_scan: @@ -960,15 +960,8 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') file_paths = self._extract_file_paths_from_wfp(wfp) - all_purls = [] - if self.scan_settings and file_paths: - seen = set() - for fp in file_paths: - for p in self.scan_settings.get_matching_purls(fp): - if p not in seen: - seen.add(p) - all_purls.append(p) - sbom = self.scan_settings.build_sbom_payload(all_purls) if self.scan_settings else None + contexts = [self.scan_settings.get_sbom_context(fp) for fp in file_paths] if self.scan_settings and file_paths else [] + sbom = SbomContext.union(contexts).to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 wfp = '' @@ -985,15 +978,8 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 wfp += scan_block # Store the WFP for the current file if wfp: file_paths = self._extract_file_paths_from_wfp(wfp) - all_purls = [] - if self.scan_settings and file_paths: - seen = set() - for fp in file_paths: - for p in self.scan_settings.get_matching_purls(fp): - if p not in seen: - seen.add(p) - all_purls.append(p) - sbom = self.scan_settings.build_sbom_payload(all_purls) if self.scan_settings else None + contexts = [self.scan_settings.get_sbom_context(fp) for fp in file_paths] if self.scan_settings and file_paths else [] + sbom = SbomContext.union(contexts).to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 @@ -1014,15 +1000,8 @@ def scan_wfp(self, wfp: str) -> bool: raise Exception('ERROR: Please specify a WFP to scan') raw_output = '{\n' file_paths = self._extract_file_paths_from_wfp(wfp) - all_purls = [] - if self.scan_settings and file_paths: - seen = set() - for fp in file_paths: - for p in self.scan_settings.get_matching_purls(fp): - if p not in seen: - seen.add(p) - all_purls.append(p) - sbom = self.scan_settings.build_sbom_payload(all_purls) if self.scan_settings else None + contexts = [self.scan_settings.get_sbom_context(fp) for fp in file_paths] if self.scan_settings and file_paths else [] + sbom = SbomContext.union(contexts).to_payload() scan_resp = self.scanoss_api.scan(wfp, sbom=sbom) if scan_resp is not None: for key, value in scan_resp.items(): diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 95b3e12b..f2b597e7 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -110,6 +110,44 @@ def from_dict(cls, data: dict) -> 'ReplaceRule': ) +@dataclass(frozen=True) +class SbomContext: + """SBOM context for a file or batch of files.""" + + purls: tuple # Use tuple for hashability (frozen dataclass) + scan_type: Optional[str] + + def to_payload(self) -> Optional[dict]: + """Convert to API payload dict.""" + if not self.purls or not self.scan_type: + return None + components = [{'purl': p} for p in self.purls] + return { + 'assets': json.dumps({'components': components}), + 'scan_type': self.scan_type, + } + + @classmethod + def empty(cls) -> 'SbomContext': + """Return empty context (no purls, no scan_type).""" + return cls(purls=(), scan_type=None) + + @classmethod + def union(cls, contexts: list) -> 'SbomContext': + """Merge multiple contexts: union of purls, first non-None scan_type wins.""" + all_purls = [] + scan_type = None + seen = set() + for ctx in contexts: + if scan_type is None and ctx.scan_type: + scan_type = ctx.scan_type + for p in ctx.purls: + if p not in seen: + seen.add(p) + all_purls.append(p) + return cls(purls=tuple(all_purls), scan_type=scan_type) + + def find_best_match(result_path: str, result_purls: List[str], entries: List[BomEntry]) -> Optional[BomEntry]: """ Find the highest-priority BOM entry that matches a result. @@ -386,9 +424,9 @@ def _get_sbom_assets(self): return self.normalize_bom_entries(self.get_bom_remove()) - def get_matching_purls(self, file_path: str) -> list: + def _get_purls_for_path(self, file_path: str, entries: List[BomEntry]) -> list: """ - Get purls matching a file path, ordered by specificity (most specific first). + Extract matching purls from entries for a given file path. Purl-only entries (no path) are always included. Path-scoped entries are included only if the file matches. @@ -396,22 +434,16 @@ def get_matching_purls(self, file_path: str) -> list: Args: file_path: File path to check + entries: List of BomEntry to check against Returns: List of purl strings ordered by specificity (most specific first) """ - if not self.data: - return [] - - include_entries = self.get_bom_include() - exclude_entries = self.get_bom_exclude() - bom_entries = include_entries or exclude_entries - - if not bom_entries: + if not entries: return [] purl_scores = {} - for entry in bom_entries: + for entry in entries: entry_purl = entry.purl or '' if not entry_purl: continue @@ -425,27 +457,35 @@ def get_matching_purls(self, file_path: str) -> list: # Sort by score descending (most specific first) return sorted(purl_scores.keys(), key=lambda p: -purl_scores[p]) - def build_sbom_payload(self, purls: list) -> Optional[dict]: + def get_sbom_context(self, file_path: str) -> SbomContext: """ - Build the SBOM payload dict from an ordered list of purls. + Get SBOM context matching a file path. + + Logic: + 1. Try include rules first - if any match, return SbomContext with 'identify' + 2. Fall back to exclude rules - if any match, return SbomContext with 'blacklist' + 3. If nothing matches, return empty SbomContext Args: - purls: List of purl strings (order is preserved) + file_path: File path to check Returns: - SBOM payload dict with 'assets' and 'scan_type', or None if no purls + SbomContext with purls ordered by specificity """ - if not purls: - return None + if not self.data: + return SbomContext.empty() - include_entries = self.get_bom_include() - scan_type = 'identify' if include_entries else 'blacklist' + # Try include first + include_purls = self._get_purls_for_path(file_path, self.get_bom_include()) + if include_purls: + return SbomContext(purls=tuple(include_purls), scan_type='identify') - components = [{'purl': p} for p in purls] # preserve order - return { - 'assets': json.dumps({'components': components}), - 'scan_type': scan_type, - } + # Fall back to exclude + exclude_purls = self._get_purls_for_path(file_path, self.get_bom_exclude()) + if exclude_purls: + return SbomContext(purls=tuple(exclude_purls), scan_type='blacklist') + + return SbomContext.empty() @staticmethod def normalize_bom_entries(bom_entries: List[BomEntry]) -> list: diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index dc09595c..1b88eb64 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -34,6 +34,7 @@ from scanoss.scanoss_settings import ( BomEntry, ReplaceRule, + SbomContext, ScanossSettings, find_best_match, ) @@ -323,9 +324,9 @@ def _make_settings(self, settings_data: dict) -> ScanossSettings: settings.data = settings_data return settings - # -- get_matching_purls tests -- + # -- get_sbom_context tests -- - def test_get_matching_purls_purl_only(self): + def test_get_sbom_context_purl_only(self): """Purl-only entries should match any file path""" settings = self._make_settings({ 'bom': { @@ -335,11 +336,12 @@ def test_get_matching_purls_purl_only(self): ], } }) - result = settings.get_matching_purls('anything/file.c') - self.assertIsInstance(result, list) - self.assertEqual(set(result), {'pkg:npm/global', 'pkg:npm/other'}) + context = settings.get_sbom_context('anything/file.c') + self.assertIsInstance(context, SbomContext) + self.assertEqual(set(context.purls), {'pkg:npm/global', 'pkg:npm/other'}) + self.assertEqual(context.scan_type, 'identify') - def test_get_matching_purls_folder_scoped(self): + def test_get_sbom_context_folder_scoped(self): """Folder-scoped entries should only match files under that folder""" settings = self._make_settings({ 'bom': { @@ -350,30 +352,33 @@ def test_get_matching_purls_folder_scoped(self): } }) # File under vendor/ gets both purls - result_vendor = settings.get_matching_purls('src/vendor/lib.c') - self.assertEqual(set(result_vendor), {'pkg:npm/global', 'pkg:npm/vendor-lib'}) + context_vendor = settings.get_sbom_context('src/vendor/lib.c') + self.assertEqual(set(context_vendor.purls), {'pkg:npm/global', 'pkg:npm/vendor-lib'}) + self.assertEqual(context_vendor.scan_type, 'identify') # File outside vendor/ gets only global - result_other = settings.get_matching_purls('src/core/main.c') - self.assertEqual(result_other, ['pkg:npm/global']) + context_other = settings.get_sbom_context('src/core/main.c') + self.assertEqual(context_other.purls, ('pkg:npm/global',)) - def test_get_matching_purls_no_data(self): - """Should return empty list when no data""" + def test_get_sbom_context_no_data(self): + """Should return empty SbomContext when no data""" settings = self._make_settings({}) - result = settings.get_matching_purls('src/main.c') - self.assertEqual(result, []) + context = settings.get_sbom_context('src/main.c') + self.assertEqual(context.purls, ()) + self.assertIsNone(context.scan_type) - def test_get_matching_purls_no_entries(self): - """Should return empty list when no include/exclude entries""" + def test_get_sbom_context_no_entries(self): + """Should return empty SbomContext when no include/exclude entries""" settings = self._make_settings({ 'bom': { 'include': [], 'exclude': [], } }) - result = settings.get_matching_purls('src/main.c') - self.assertEqual(result, []) + context = settings.get_sbom_context('src/main.c') + self.assertEqual(context.purls, ()) + self.assertIsNone(context.scan_type) - def test_get_matching_purls_deduplicates(self): + def test_get_sbom_context_deduplicates(self): """Should not duplicate purls when multiple entries match same purl""" settings = self._make_settings({ 'bom': { @@ -383,10 +388,10 @@ def test_get_matching_purls_deduplicates(self): ], } }) - result = settings.get_matching_purls('src/main.c') - self.assertEqual(result.count('pkg:npm/vue'), 1) + context = settings.get_sbom_context('src/main.c') + self.assertEqual(context.purls.count('pkg:npm/vue'), 1) - def test_get_matching_purls_ordered_by_specificity(self): + def test_get_sbom_context_ordered_by_specificity(self): """Should return purls ordered by specificity (most specific first)""" settings = self._make_settings({ 'bom': { @@ -397,11 +402,11 @@ def test_get_matching_purls_ordered_by_specificity(self): ], } }) - result = settings.get_matching_purls('src/vendor/lib.c') + context = settings.get_sbom_context('src/vendor/lib.c') # Most specific first: vendor-lib (15), src-lib (8), global (2) - self.assertEqual(result, ['pkg:npm/vendor-lib', 'pkg:npm/src-lib', 'pkg:npm/global']) + self.assertEqual(context.purls, ('pkg:npm/vendor-lib', 'pkg:npm/src-lib', 'pkg:npm/global')) - def test_get_matching_purls_file_path_most_specific(self): + def test_get_sbom_context_file_path_most_specific(self): """File path should be more specific than folder path""" settings = self._make_settings({ 'bom': { @@ -411,60 +416,102 @@ def test_get_matching_purls_file_path_most_specific(self): ], } }) - result = settings.get_matching_purls('src/main.c') + context = settings.get_sbom_context('src/main.c') # File path (10 chars) more specific than folder path (4 chars) - self.assertEqual(result[0], 'pkg:npm/file-lib') + self.assertEqual(context.purls[0], 'pkg:npm/file-lib') - # -- build_sbom_payload tests -- + # -- SbomContext.to_payload tests -- - def test_build_sbom_payload_identify(self): - """Should return identify scan type for include entries""" - settings = self._make_settings({ - 'bom': { - 'include': [{'purl': 'pkg:npm/vue'}], - } - }) - result = settings.build_sbom_payload(['pkg:npm/vue', 'pkg:npm/react']) + def test_sbom_context_to_payload_identify(self): + """Should return identify scan type when scan_type is 'identify'""" + context = SbomContext(purls=('pkg:npm/vue', 'pkg:npm/react'), scan_type='identify') + result = context.to_payload() self.assertIsNotNone(result) self.assertEqual(result['scan_type'], 'identify') assets = json.loads(result['assets']) # Order should be preserved self.assertEqual(assets['components'], [{'purl': 'pkg:npm/vue'}, {'purl': 'pkg:npm/react'}]) - def test_build_sbom_payload_blacklist(self): - """Should return blacklist scan type for exclude entries""" - settings = self._make_settings({ - 'bom': { - 'exclude': [{'purl': 'pkg:npm/excluded'}], - } - }) - result = settings.build_sbom_payload(['pkg:npm/excluded']) + def test_sbom_context_to_payload_blacklist(self): + """Should return blacklist scan type when scan_type is 'blacklist'""" + context = SbomContext(purls=('pkg:npm/excluded',), scan_type='blacklist') + result = context.to_payload() self.assertIsNotNone(result) self.assertEqual(result['scan_type'], 'blacklist') - def test_build_sbom_payload_empty_purls(self): - """Should return None for empty purls list""" - settings = self._make_settings({ - 'bom': { - 'include': [{'purl': 'pkg:npm/vue'}], - } - }) - result = settings.build_sbom_payload([]) + def test_sbom_context_to_payload_empty_purls(self): + """Should return None for empty purls tuple""" + context = SbomContext(purls=(), scan_type='identify') + result = context.to_payload() + self.assertIsNone(result) + + def test_sbom_context_to_payload_none_scan_type(self): + """Should return None when scan_type is None""" + context = SbomContext(purls=('pkg:npm/vue',), scan_type=None) + result = context.to_payload() self.assertIsNone(result) - def test_build_sbom_payload_preserves_order(self): + def test_sbom_context_to_payload_preserves_order(self): """Should preserve the order of purls in the payload""" - settings = self._make_settings({ - 'bom': { - 'include': [{'purl': 'pkg:npm/a'}], - } - }) - purls = ['pkg:npm/c', 'pkg:npm/a', 'pkg:npm/b'] - result = settings.build_sbom_payload(purls) + purls = ('pkg:npm/c', 'pkg:npm/a', 'pkg:npm/b') + context = SbomContext(purls=purls, scan_type='identify') + result = context.to_payload() assets = json.loads(result['assets']) - self.assertEqual([c['purl'] for c in assets['components']], purls) - - # -- Integration tests (get_matching_purls + build_sbom_payload) -- + self.assertEqual([c['purl'] for c in assets['components']], list(purls)) + + def test_sbom_context_empty(self): + """SbomContext.empty() should return empty context""" + context = SbomContext.empty() + self.assertEqual(context.purls, ()) + self.assertIsNone(context.scan_type) + self.assertIsNone(context.to_payload()) + + # -- SbomContext.union tests -- + + def test_sbom_context_union_empty_list(self): + """Union of empty list should return empty context""" + result = SbomContext.union([]) + self.assertEqual(result.purls, ()) + self.assertIsNone(result.scan_type) + + def test_sbom_context_union_single(self): + """Union of single context should return equivalent context""" + ctx = SbomContext(purls=('pkg:npm/a',), scan_type='identify') + result = SbomContext.union([ctx]) + self.assertEqual(result.purls, ('pkg:npm/a',)) + self.assertEqual(result.scan_type, 'identify') + + def test_sbom_context_union_multiple_same_type(self): + """Union of multiple contexts with same scan_type""" + ctx1 = SbomContext(purls=('pkg:npm/a',), scan_type='identify') + ctx2 = SbomContext(purls=('pkg:npm/b',), scan_type='identify') + result = SbomContext.union([ctx1, ctx2]) + self.assertEqual(result.purls, ('pkg:npm/a', 'pkg:npm/b')) + self.assertEqual(result.scan_type, 'identify') + + def test_sbom_context_union_mixed_types_first_wins(self): + """Union of mixed scan_types: first non-None wins""" + ctx1 = SbomContext(purls=('pkg:npm/a',), scan_type='identify') + ctx2 = SbomContext(purls=('pkg:npm/b',), scan_type='blacklist') + result = SbomContext.union([ctx1, ctx2]) + self.assertEqual(result.purls, ('pkg:npm/a', 'pkg:npm/b')) + self.assertEqual(result.scan_type, 'identify') + + def test_sbom_context_union_deduplicates_purls(self): + """Union should deduplicate purls while preserving order""" + ctx1 = SbomContext(purls=('pkg:npm/a', 'pkg:npm/b'), scan_type='identify') + ctx2 = SbomContext(purls=('pkg:npm/b', 'pkg:npm/c'), scan_type='identify') + result = SbomContext.union([ctx1, ctx2]) + self.assertEqual(result.purls, ('pkg:npm/a', 'pkg:npm/b', 'pkg:npm/c')) + + def test_sbom_context_union_skips_none_scan_type(self): + """Union should skip None scan_types and use first non-None""" + ctx1 = SbomContext(purls=('pkg:npm/a',), scan_type=None) + ctx2 = SbomContext(purls=('pkg:npm/b',), scan_type='blacklist') + result = SbomContext.union([ctx1, ctx2]) + self.assertEqual(result.scan_type, 'blacklist') + + # -- Integration tests (get_sbom_context + to_payload) -- def test_folder_scoped_entry_included_when_matching(self): """Folder-scoped entry should be included when file matches""" @@ -475,8 +522,8 @@ def test_folder_scoped_entry_included_when_matching(self): ], } }) - purls = settings.get_matching_purls('src/vendor/lib.c') - result = settings.build_sbom_payload(purls) + context = settings.get_sbom_context('src/vendor/lib.c') + result = context.to_payload() self.assertIsNotNone(result) assets = json.loads(result['assets']) self.assertEqual([c['purl'] for c in assets['components']], ['pkg:npm/vue']) @@ -490,8 +537,8 @@ def test_folder_scoped_entry_excluded_when_no_match(self): ], } }) - purls = settings.get_matching_purls('lib/other.c') - result = settings.build_sbom_payload(purls) + context = settings.get_sbom_context('lib/other.c') + result = context.to_payload() self.assertIsNone(result) def test_mixed_purl_only_and_scoped(self): @@ -505,10 +552,31 @@ def test_mixed_purl_only_and_scoped(self): ], } }) - purls = settings.get_matching_purls('src/vendor/file.c') - self.assertIn('pkg:npm/global-lib', purls) - self.assertIn('pkg:npm/vendor-lib', purls) - self.assertNotIn('pkg:npm/lib-only', purls) + context = settings.get_sbom_context('src/vendor/file.c') + self.assertIn('pkg:npm/global-lib', context.purls) + self.assertIn('pkg:npm/vendor-lib', context.purls) + self.assertNotIn('pkg:npm/lib-only', context.purls) + + def test_include_no_match_falls_back_to_exclude(self): + """When include rules exist but don't match, should fall back to exclude rules""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'path': 'vendor/', 'purl': 'pkg:npm/react'}, + ], + 'exclude': [ + {'purl': 'pkg:npm/lodash'}, + ], + } + }) + # File in vendor/ should use include rules + context_vendor = settings.get_sbom_context('vendor/lib.c') + self.assertEqual(context_vendor.purls, ('pkg:npm/react',)) + self.assertEqual(context_vendor.scan_type, 'identify') + # File outside vendor/ should fall back to exclude rules + context_other = settings.get_sbom_context('src/app.js') + self.assertEqual(context_other.purls, ('pkg:npm/lodash',)) + self.assertEqual(context_other.scan_type, 'blacklist') class TestExtractFilePathsFromWfp(unittest.TestCase): From c257cdcfe3a047dcd9c14b11c4c0d7c1b3222efe Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 22:28:16 +0100 Subject: [PATCH 439/489] refactor(settings): remove unused SBOM construction logic and duplicate handling methods --- src/scanoss/scanoss_settings.py | 86 --------------------------------- 1 file changed, 86 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index f2b597e7..1fc8fb5e 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -375,55 +375,6 @@ def get_bom_replace(self) -> List[ReplaceRule]: raw = self._get_bom().get('replace', []) return [ReplaceRule.from_dict(entry) for entry in raw] - def get_sbom(self): - """ - Get the SBOM to be sent to the SCANOSS API - Returns: - dict: SBOM request payload - """ - if not self.data: - return None - return { - 'assets': json.dumps(self._get_sbom_assets()), - 'scan_type': self.scan_type, - } - - def _get_sbom_assets(self): - """ - Get the SBOM assets - Returns: - List: List of SBOM assets - """ - - if self.settings_file_type == 'new': - if len(self.get_bom_include()): - self.scan_type = 'identify' - include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include())) - return {"components": include_bom_entries} - elif len(self.get_bom_exclude()): - self.scan_type = 'blacklist' - exclude_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_exclude())) - return {"components": exclude_bom_entries} - - if self.settings_file_type == 'legacy' and self.scan_type == 'identify': # sbom-identify.json - include_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_include())) - replace_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_replace())) - self.print_debug( - f"Scan type set to 'identify'. Adding {len(include_bom_entries) + len(replace_bom_entries)} components as context to the scan. \n" # noqa: E501 - f'From Include list: {[entry["purl"] for entry in include_bom_entries]} \n' - f'From Replace list: {[entry["purl"] for entry in replace_bom_entries]} \n' - ) - return include_bom_entries + replace_bom_entries - - if self.settings_file_type == 'legacy' and self.scan_type == 'blacklist': # sbom-identify.json - exclude_bom_entries = self._remove_duplicates(self.normalize_bom_entries(self.get_bom_exclude())) - self.print_debug( - f"Scan type set to 'blacklist'. Adding {len(exclude_bom_entries)} components as context to the scan. \n" # noqa: E501 - f'From Exclude list: {[entry["purl"] for entry in exclude_bom_entries]} \n') - return exclude_bom_entries - - return self.normalize_bom_entries(self.get_bom_remove()) - def _get_purls_for_path(self, file_path: str, entries: List[BomEntry]) -> list: """ Extract matching purls from entries for a given file path. @@ -487,43 +438,6 @@ def get_sbom_context(self, file_path: str) -> SbomContext: return SbomContext.empty() - @staticmethod - def normalize_bom_entries(bom_entries: List[BomEntry]) -> list: - """ - Normalize the BOM entries by extracting only the purl field. - - Args: - bom_entries: List of BOM entries - Returns: - List of dicts with only the purl field (for API payload) - """ - normalized_bom_entries = [] - for entry in bom_entries: - normalized_bom_entries.append( - { - 'purl': entry.purl or '', - } - ) - return normalized_bom_entries - - @staticmethod - def _remove_duplicates(bom_entries: list) -> list: - """ - Remove duplicate BOM entries based on purl field. - Args: - bom_entries: List of normalized BOM entry dicts - Returns: - List of unique BOM entries - """ - already_added = set() - unique_entries = [] - for entry in bom_entries: - entry_tuple = tuple(entry.items()) - if entry_tuple not in already_added: - already_added.add(entry_tuple) - unique_entries.append(entry) - return unique_entries - def is_legacy(self): """Check if the settings file is legacy""" return self.settings_file_type == 'legacy' From 26eef470aee12a21915a7b5dbb4445dc4a2effd7 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 22:47:54 +0100 Subject: [PATCH 440/489] refactor(settings): update SBOM context matching to support legacy files and new format handling --- src/scanoss/scanoss_settings.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 1fc8fb5e..17702a9e 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -413,9 +413,8 @@ def get_sbom_context(self, file_path: str) -> SbomContext: Get SBOM context matching a file path. Logic: - 1. Try include rules first - if any match, return SbomContext with 'identify' - 2. Fall back to exclude rules - if any match, return SbomContext with 'blacklist' - 3. If nothing matches, return empty SbomContext + - Legacy files: use self.scan_type set during file load + - New format: try include rules first, fall back to exclude rules Args: file_path: File path to check @@ -426,12 +425,20 @@ def get_sbom_context(self, file_path: str) -> SbomContext: if not self.data: return SbomContext.empty() - # Try include first + # Legacy files: use self.scan_type set during file load (--identify or --ignore flag) + if self.is_legacy(): + raw = self._get_bom() + entries = [BomEntry.from_dict(entry) for entry in raw] + purls = self._get_purls_for_path(file_path, entries) + if purls: + return SbomContext(purls=tuple(purls), scan_type=self.scan_type) + return SbomContext.empty() + + # New format: try include first, then exclude include_purls = self._get_purls_for_path(file_path, self.get_bom_include()) if include_purls: return SbomContext(purls=tuple(include_purls), scan_type='identify') - # Fall back to exclude exclude_purls = self._get_purls_for_path(file_path, self.get_bom_exclude()) if exclude_purls: return SbomContext(purls=tuple(exclude_purls), scan_type='blacklist') From 2b763dc681beec07a4d4e68efa780d32f9bd5382 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Feb 2026 23:04:44 +0100 Subject: [PATCH 441/489] refactor(scanner): remove global SBOM context and simplify per-request SBOM handling --- src/scanoss/scanossapi.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 6314d9fe..5c2df315 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -185,15 +185,14 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None, sbom: dict = :param wfp: WFP to scan :param context: Context to help with identification :param scan_id: ID of the scan being run (usually thread id) - :param sbom: Per-request SBOM context (overrides global self.sbom if provided) + :param sbom: Per-request SBOM context :return: JSON result object """ request_id = str(uuid.uuid4()) form_data = {} - effective_sbom = sbom if sbom is not None else self.sbom - if effective_sbom: - form_data['type'] = effective_sbom.get('scan_type') - form_data['assets'] = effective_sbom.get('assets') + if sbom: + form_data['type'] = sbom.get('scan_type') + form_data['assets'] = sbom.get('assets') if self.scan_format: form_data['format'] = self.scan_format if self.flags: From da909d4c6f1382dee0b2e1d4dc899f3b79f94c97 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 6 Feb 2026 17:34:41 +0100 Subject: [PATCH 442/489] refactor(scanner, settings): replace `scan_settings` with `scanoss_settings` and enhance SBOM context handling --- src/scanoss/scanner.py | 37 ++++++++++++++++++------ src/scanoss/scanoss_settings.py | 49 ++++++++++++++++++++++++++++++++ src/scanoss/scanpostprocessor.py | 2 ++ tests/test_bom_path_matching.py | 2 +- 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index f19c2f8b..89b4e9a8 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -234,7 +234,7 @@ def __init__( # noqa: PLR0913, PLR0915 ScanPostProcessor(scanoss_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None ) self._per_batch_sbom = ( - scan_settings is not None and scan_settings.has_path_scoped_bom_entries() + scanoss_settings is not None and scanoss_settings.has_path_scoped_bom_entries() ) self._maybe_set_api_sbom() @@ -489,7 +489,10 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 wfp_size = len(wfp.encode('utf-8')) # Compute SBOM context for this file - file_context = self.scan_settings.get_sbom_context(to_scan_file) if self.scan_settings else SbomContext.empty() + file_context = ( + self.scanoss_settings.get_sbom_context(to_scan_file) + if self.scanoss_settings else SbomContext.empty() + ) # FLUSH: Context changed (different purls or scan_type) if scan_block != '' and batch_context is not None and file_context != batch_context: @@ -712,7 +715,10 @@ def scan_file(self, file: str) -> bool: wfp = self.winnowing.wfp_for_file(file, file) if wfp is not None and wfp != '': if self.threaded_scan: - file_context = self.scan_settings.get_sbom_context(file) if self.scan_settings else SbomContext.empty() + file_context = ( + self.scanoss_settings.get_sbom_context(file) + if self.scanoss_settings else SbomContext.empty() + ) sbom = file_context.to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) # Submit the WFP for scanning self.print_debug(f'Scanning {file}...') @@ -778,7 +784,10 @@ def scan_files(self, files: []) -> bool: # noqa: PLR0912, PLR0915 wfp_size = len(wfp.encode('utf-8')) # Compute SBOM context for this file - file_context = self.scan_settings.get_sbom_context(file) if self.scan_settings else SbomContext.empty() + file_context = ( + self.scanoss_settings.get_sbom_context(file) + if self.scanoss_settings else SbomContext.empty() + ) # FLUSH: Context changed (different purls or scan_type) if batch_context is not None and file_context != batch_context and scan_block != '': @@ -882,7 +891,10 @@ def scan_contents(self, filename: str, contents: bytes) -> bool: wfp = self.winnowing.wfp_for_contents(filename, False, contents) if wfp is not None and wfp != '': if self.threaded_scan: - file_context = self.scan_settings.get_sbom_context(filename) if self.scan_settings else SbomContext.empty() + file_context = ( + self.scanoss_settings.get_sbom_context(filename) + if self.scanoss_settings else SbomContext.empty() + ) sbom = file_context.to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) # Submit the WFP for scanning self.print_debug(f'Scanning {filename}...') @@ -960,7 +972,10 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') file_paths = self._extract_file_paths_from_wfp(wfp) - contexts = [self.scan_settings.get_sbom_context(fp) for fp in file_paths] if self.scan_settings and file_paths else [] + contexts = ( + [self.scanoss_settings.get_sbom_context(fp) for fp in file_paths] + if self.scanoss_settings and file_paths else [] + ) sbom = SbomContext.union(contexts).to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 @@ -978,7 +993,10 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 wfp += scan_block # Store the WFP for the current file if wfp: file_paths = self._extract_file_paths_from_wfp(wfp) - contexts = [self.scan_settings.get_sbom_context(fp) for fp in file_paths] if self.scan_settings and file_paths else [] + contexts = ( + [self.scanoss_settings.get_sbom_context(fp) for fp in file_paths] + if self.scanoss_settings and file_paths else [] + ) sbom = SbomContext.union(contexts).to_payload() self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 @@ -1000,7 +1018,10 @@ def scan_wfp(self, wfp: str) -> bool: raise Exception('ERROR: Please specify a WFP to scan') raw_output = '{\n' file_paths = self._extract_file_paths_from_wfp(wfp) - contexts = [self.scan_settings.get_sbom_context(fp) for fp in file_paths] if self.scan_settings and file_paths else [] + contexts = ( + [self.scanoss_settings.get_sbom_context(fp) for fp in file_paths] + if self.scanoss_settings and file_paths else [] + ) sbom = SbomContext.union(contexts).to_payload() scan_resp = self.scanoss_api.scan(wfp, sbom=sbom) if scan_resp is not None: diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 17702a9e..f68543e7 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -375,6 +375,21 @@ def get_bom_replace(self) -> List[ReplaceRule]: raw = self._get_bom().get('replace', []) return [ReplaceRule.from_dict(entry) for entry in raw] + def has_path_scoped_bom_entries(self) -> bool: + """ + Check if there are any BOM entries with path-scoped rules. + + Returns: + bool: True if any include or exclude entry has a path field set + """ + for entry in self.get_bom_include(): + if entry.path: + return True + for entry in self.get_bom_exclude(): + if entry.path: + return True + return False + def _get_purls_for_path(self, file_path: str, entries: List[BomEntry]) -> list: """ Extract matching purls from entries for a given file path. @@ -445,6 +460,40 @@ def get_sbom_context(self, file_path: str) -> SbomContext: return SbomContext.empty() + def get_sbom(self) -> 'dict | None': + """ + Get global SBOM payload (for purl-only entries without path scope). + + This returns the SBOM context using an empty path, which includes + all purl-only entries (entries without a path field). + + Returns: + dict: API payload with 'assets' and 'scan_type' keys, or None if no entries + """ + # Use empty path to get purl-only entries + context = self.get_sbom_context('') + return context.to_payload() + + def get_sbom_for_batch(self, batch_file_paths: list) -> 'dict | None': + """ + Get SBOM payload for a batch of files. + + Computes the union of SBOM contexts for all files in the batch. + All matching purls are included, deduplicated. + + Args: + batch_file_paths: List of file paths in the batch + + Returns: + dict: API payload with 'assets' and 'scan_type' keys, or None if no entries + """ + if not batch_file_paths: + return None + + contexts = [self.get_sbom_context(path) for path in batch_file_paths] + merged = SbomContext.union(contexts) + return merged.to_payload() + def is_legacy(self): """Check if the settings file is legacy""" return self.settings_file_type == 'legacy' diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index c751ddd2..7ba06534 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -107,6 +107,8 @@ def post_process(self): Returns: dict: Processed results """ + if not self.scanoss_settings: + return self.results if self.scanoss_settings.is_legacy(): self.print_stderr( 'Legacy settings file detected. Post-processing is not supported for legacy settings file.' diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index 1b88eb64..b0a8d89f 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -634,7 +634,7 @@ def _create_scanner(self, settings=None): (scanner, mock_post) tuple """ scanner = Scanner( - scan_settings=settings, + scanoss_settings=settings, nb_threads=1, quiet=True, scan_options=3, # FILES + SNIPPETS, no deps From 0202cb738b127deb701873c63d0fe2d8d014d937 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 6 Feb 2026 18:07:53 +0100 Subject: [PATCH 443/489] refactor(scanner): remove unused `_get_batch_sbom` method and associated code --- src/scanoss/scanner.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 89b4e9a8..5935663e 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -248,21 +248,6 @@ def _maybe_set_api_sbom(self): if sbom: self.scanoss_api.set_sbom(sbom) - def _get_batch_sbom(self, batch_file_paths: List[str]) -> 'dict | None': - """ - Compute SBOM context for a specific batch of files. - Purl-only entries are always included; path-scoped entries - are included only when a batch file matches. - - Args: - batch_file_paths: List of file paths in the current batch - Returns: - SBOM payload dict or None - """ - if not self.scanoss_settings: - return None - return self.scanoss_settings.get_sbom_for_batch(batch_file_paths) - @staticmethod def _extract_file_paths_from_wfp(wfp: str) -> List[str]: """ From 09251a3aaf5799c3b3d6129c184c7594b3fc29f5 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 6 Feb 2026 18:36:28 +0100 Subject: [PATCH 444/489] refactor(scanner): replace magic number with `WFP_FILE_PARTS` constant for clarity --- src/scanoss/scanner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 5935663e..a6c952c3 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -61,6 +61,7 @@ from .winnowing import Winnowing WFP_FILE_START = 'file=' +WFP_FILE_PARTS = 3 # WFP format: file=,, MAX_POST_SIZE = 64 * 1024 # 64k Max post size @@ -263,7 +264,7 @@ def _extract_file_paths_from_wfp(wfp: str) -> List[str]: for line in wfp.split('\n'): if line.startswith(WFP_FILE_START): parts = line[len(WFP_FILE_START):].split(',', 2) - if len(parts) >= 3: + if len(parts) >= WFP_FILE_PARTS: paths.append(parts[2].strip()) return paths From 9916895e56ca29382f366f410bcbab04af566448 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 6 Feb 2026 19:17:20 +0100 Subject: [PATCH 445/489] feat(scanner): support folder-level BOM filtering in scan_wfp_file_threaded --- src/scanoss/scanner.py | 76 ++++++++++++++++++--------------- tests/test_bom_path_matching.py | 24 ----------- 2 files changed, 42 insertions(+), 58 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index a6c952c3..10b164ba 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -250,23 +250,22 @@ def _maybe_set_api_sbom(self): self.scanoss_api.set_sbom(sbom) @staticmethod - def _extract_file_paths_from_wfp(wfp: str) -> List[str]: + def _extract_file_path_from_line(line: str) -> str: """ - Extract file paths from a WFP string. + Extract file path from a single WFP line. WFP file lines have the format: file=,, Args: - wfp: WFP string + line: Single WFP line starting with 'file=' Returns: - List of file paths + File path or empty string if not found """ - paths = [] - for line in wfp.split('\n'): - if line.startswith(WFP_FILE_START): - parts = line[len(WFP_FILE_START):].split(',', 2) - if len(parts) >= WFP_FILE_PARTS: - paths.append(parts[2].strip()) - return paths + if not line.startswith(WFP_FILE_START): + return '' + parts = line[len(WFP_FILE_START):].split(',', 2) + if len(parts) >= WFP_FILE_PARTS: + return parts[2].strip() + return '' @staticmethod def _merge_cli_with_settings(cli_value, settings_value): @@ -923,7 +922,7 @@ def scan_wfp_with_options(self, wfp_file: str, deps_file: str, file_map: dict = success = False return success - def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 + def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912, PLR0915 """ Scan the contents of the specified WFP file (threaded) :param wfp_file: WFP file to scan @@ -941,32 +940,48 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 scan_started = False wfp = '' scan_block = '' + batch_context = None # Track SBOM context for the current batch with open(wfp_file) as f: # Parse the WFP file for line in f: if line.startswith(WFP_FILE_START): + # First, add previous file's content to wfp before checking context if scan_block: - wfp += scan_block # Store the WFP for the current file + wfp += scan_block cur_size = len(wfp.encode('utf-8')) + + file_path = Scanner._extract_file_path_from_line(line) + file_context = ( + self.scanoss_settings.get_sbom_context(file_path) + if self.scanoss_settings else SbomContext.empty() + ) + + # FLUSH: Context changed (different purls or scan_type) + if wfp and batch_context is not None and file_context != batch_context: + sbom = batch_context.to_payload() + self.threaded_scan.queue_add(wfp, sbom=sbom) + queue_size += 1 + wfp = '' + wfp_file_count = 0 + cur_size = 0 + batch_context = None + scan_block = line # Start storing the next file + batch_context = file_context file_count += 1 wfp_file_count += 1 else: scan_block += line # Store the rest of the WFP for this file l_size = cur_size + len(scan_block.encode('utf-8')) - # Hit the max post size, so sending the current batch and continue processing + # FLUSH: Hit the max post size if (wfp_file_count > self.post_file_count or l_size >= self.max_post_size) and wfp: if self.debug and cur_size > self.max_post_size: Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') - file_paths = self._extract_file_paths_from_wfp(wfp) - contexts = ( - [self.scanoss_settings.get_sbom_context(fp) for fp in file_paths] - if self.scanoss_settings and file_paths else [] - ) - sbom = SbomContext.union(contexts).to_payload() + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 wfp = '' wfp_file_count = 0 + batch_context = None if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do scan_started = True if not self.threaded_scan.run(wait=False): @@ -978,12 +993,7 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 if scan_block: wfp += scan_block # Store the WFP for the current file if wfp: - file_paths = self._extract_file_paths_from_wfp(wfp) - contexts = ( - [self.scanoss_settings.get_sbom_context(fp) for fp in file_paths] - if self.scanoss_settings and file_paths else [] - ) - sbom = SbomContext.union(contexts).to_payload() + sbom = batch_context.to_payload() if batch_context else None self.threaded_scan.queue_add(wfp, sbom=sbom) queue_size += 1 @@ -993,7 +1003,11 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912 def scan_wfp(self, wfp: str) -> bool: """ - Send the specified (single) WFP to ScanOSS for identification + Send the specified (single) WFP to ScanOSS for identification. + + NOTE: This method does not support the scanoss settings file (scanoss.json). + For folder-level BOM filtering, use scan_wfp_with_options instead. + Parameters ---------- wfp: str @@ -1003,13 +1017,7 @@ def scan_wfp(self, wfp: str) -> bool: if not wfp: raise Exception('ERROR: Please specify a WFP to scan') raw_output = '{\n' - file_paths = self._extract_file_paths_from_wfp(wfp) - contexts = ( - [self.scanoss_settings.get_sbom_context(fp) for fp in file_paths] - if self.scanoss_settings and file_paths else [] - ) - sbom = SbomContext.union(contexts).to_payload() - scan_resp = self.scanoss_api.scan(wfp, sbom=sbom) + scan_resp = self.scanoss_api.scan(wfp) if scan_resp is not None: for key, value in scan_resp.items(): raw_output += ' "%s":%s' % (key, json.dumps(value, indent=2)) diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index b0a8d89f..0c7f2fb3 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -579,30 +579,6 @@ def test_include_no_match_falls_back_to_exclude(self): self.assertEqual(context_other.scan_type, 'blacklist') -class TestExtractFilePathsFromWfp(unittest.TestCase): - """Test WFP file path extraction""" - - def test_extract_single_file(self): - from scanoss.scanner import Scanner - wfp = 'file=abc123,1024,src/main.c\n4=abcdef\n' - paths = Scanner._extract_file_paths_from_wfp(wfp) - self.assertEqual(paths, ['src/main.c']) - - def test_extract_multiple_files(self): - from scanoss.scanner import Scanner - wfp = ( - 'file=abc123,1024,src/main.c\n4=abcdef\n' - 'file=def456,2048,src/vendor/lib.c\n4=ghijkl\n' - ) - paths = Scanner._extract_file_paths_from_wfp(wfp) - self.assertEqual(paths, ['src/main.c', 'src/vendor/lib.c']) - - def test_extract_empty_wfp(self): - from scanoss.scanner import Scanner - paths = Scanner._extract_file_paths_from_wfp('') - self.assertEqual(paths, []) - - class TestScannerSbomPayload(unittest.TestCase): """End-to-end tests: verify Scanner sends the correct SBOM payload in HTTP POST requests""" From e436daef47b36f65e711f944dff557147858224e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Fri, 13 Feb 2026 12:39:32 +0100 Subject: [PATCH 446/489] feat: set result identified when replacing a component --- src/scanoss/scanpostprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 7ba06534..b2f927c8 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -157,7 +157,6 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di Returns: dict: Updated result """ - if self.component_info_map.get(to_replace_with_purl): result.update(self.component_info_map[to_replace_with_purl]) else: @@ -187,6 +186,7 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di result.pop('version', None) result['purl'] = [to_replace_with_purl] + result['status'] = 'identified' return result From e96a68e46a0e974536c8cd3d5af721e7b3cb36ab Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 2 Mar 2026 11:04:30 +0100 Subject: [PATCH 447/489] refactor(scanner, settings, api): remove dead init-time SBOM code Remove set_sbom(), _maybe_set_api_sbom(), has_path_scoped_bom_entries(), and get_sbom() which became dead code after scan() was refactored to accept a per-request sbom parameter. --- src/scanoss/scanner.py | 14 -------------- src/scanoss/scanoss_settings.py | 29 ----------------------------- src/scanoss/scanossapi.py | 5 ----- 3 files changed, 48 deletions(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 10b164ba..a9a16f0b 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -234,20 +234,6 @@ def __init__( # noqa: PLR0913, PLR0915 self.post_processor = ( ScanPostProcessor(scanoss_settings, debug=debug, trace=trace, quiet=quiet) if scan_settings else None ) - self._per_batch_sbom = ( - scanoss_settings is not None and scanoss_settings.has_path_scoped_bom_entries() - ) - self._maybe_set_api_sbom() - - def _maybe_set_api_sbom(self): - if not self.scanoss_settings: - return - if self._per_batch_sbom: - self.print_debug('Path-scoped BOM entries detected. SBOM context will be resolved per-batch.') - return - sbom = self.scanoss_settings.get_sbom() - if sbom: - self.scanoss_api.set_sbom(sbom) @staticmethod def _extract_file_path_from_line(line: str) -> str: diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index f68543e7..ddc20622 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -375,21 +375,6 @@ def get_bom_replace(self) -> List[ReplaceRule]: raw = self._get_bom().get('replace', []) return [ReplaceRule.from_dict(entry) for entry in raw] - def has_path_scoped_bom_entries(self) -> bool: - """ - Check if there are any BOM entries with path-scoped rules. - - Returns: - bool: True if any include or exclude entry has a path field set - """ - for entry in self.get_bom_include(): - if entry.path: - return True - for entry in self.get_bom_exclude(): - if entry.path: - return True - return False - def _get_purls_for_path(self, file_path: str, entries: List[BomEntry]) -> list: """ Extract matching purls from entries for a given file path. @@ -460,20 +445,6 @@ def get_sbom_context(self, file_path: str) -> SbomContext: return SbomContext.empty() - def get_sbom(self) -> 'dict | None': - """ - Get global SBOM payload (for purl-only entries without path scope). - - This returns the SBOM context using an empty path, which includes - all purl-only entries (entries without a path field). - - Returns: - dict: API payload with 'assets' and 'scan_type' keys, or None if no entries - """ - # Use empty path to get purl-only entries - context = self.get_sbom_context('') - return context.to_payload() - def get_sbom_for_batch(self, batch_file_paths: list) -> 'dict | None': """ Get SBOM payload for a batch of files. diff --git a/src/scanoss/scanossapi.py b/src/scanoss/scanossapi.py index 5c2df315..a6816e88 100644 --- a/src/scanoss/scanossapi.py +++ b/src/scanoss/scanossapi.py @@ -129,7 +129,6 @@ def __init__( # noqa: PLR0912, PLR0913, PLR0915 HTTPS_PROXY='http://:' """ super().__init__(debug, trace, quiet) - self.sbom = None # Scan tuning parameters self.min_snippet_hits = min_snippet_hits self.min_snippet_lines = min_snippet_lines @@ -316,10 +315,6 @@ def save_bad_req_wfp(self, scan_files, request_id, scan_id): f'Warning: Issue writing bad request file - {bad_req_file} ({ee.__class__.__name__}): {ee}' ) - def set_sbom(self, sbom): - self.sbom = sbom - return self - def _build_scan_settings_header(self) -> Optional[str]: """ Build base64-encoded JSON for x-scanoss-scan-settings header. From 0d50312475a922a0a6b3664f92c968ee1a6b29f1 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 2 Mar 2026 11:07:19 +0100 Subject: [PATCH 448/489] refactor(settings): remove unused `get_sbom_for_batch` method --- src/scanoss/scanoss_settings.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index ddc20622..17702a9e 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -445,26 +445,6 @@ def get_sbom_context(self, file_path: str) -> SbomContext: return SbomContext.empty() - def get_sbom_for_batch(self, batch_file_paths: list) -> 'dict | None': - """ - Get SBOM payload for a batch of files. - - Computes the union of SBOM contexts for all files in the batch. - All matching purls are included, deduplicated. - - Args: - batch_file_paths: List of file paths in the batch - - Returns: - dict: API payload with 'assets' and 'scan_type' keys, or None if no entries - """ - if not batch_file_paths: - return None - - contexts = [self.get_sbom_context(path) for path in batch_file_paths] - merged = SbomContext.union(contexts) - return merged.to_payload() - def is_legacy(self): """Check if the settings file is legacy""" return self.settings_file_type == 'legacy' From 92805aa753d2009a2cc7dcdee7d21f45754261c9 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 2 Mar 2026 16:31:02 +0100 Subject: [PATCH 449/489] refactor(scanner): extract `_iter_wfp_files` generator to simplify WFP parsing in `scan_wfp_file_threaded` --- src/scanoss/scanner.py | 131 +++++++++++-------- tests/test_iter_wfp_files.py | 123 ++++++++++++++++++ tests/test_scan_wfp_file_threaded.py | 185 +++++++++++++++++++++++++++ 3 files changed, 387 insertions(+), 52 deletions(-) create mode 100644 tests/test_iter_wfp_files.py create mode 100644 tests/test_scan_wfp_file_threaded.py diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index a9a16f0b..3df889b4 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -253,6 +253,34 @@ def _extract_file_path_from_line(line: str) -> str: return parts[2].strip() return '' + @staticmethod + def _iter_wfp_files(wfp_file: str): + """Yield (file_path, wfp_content) for each file entry in a WFP file. + + Parses the line-by-line WFP format into complete per-file blocks. + + Parameters + ---------- + wfp_file: str + Path to the WFP file to parse + Yields + ------ + tuple[str, str]: (file_path, wfp_content) for each file entry + """ + current_block = '' + current_path = None + with open(wfp_file) as f: + for line in f: + if line.startswith(WFP_FILE_START): + if current_block: + yield current_path, current_block + current_path = Scanner._extract_file_path_from_line(line) + current_block = line + else: + current_block += line + if current_block: + yield current_path, current_block + @staticmethod def _merge_cli_with_settings(cli_value, settings_value): """Merge CLI value with settings value (two-level priority: settings > cli). @@ -908,7 +936,7 @@ def scan_wfp_with_options(self, wfp_file: str, deps_file: str, file_map: dict = success = False return success - def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912, PLR0915 + def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0915 """ Scan the contents of the specified WFP file (threaded) :param wfp_file: WFP file to scan @@ -919,68 +947,67 @@ def scan_wfp_file_threaded(self, wfp_file: str) -> bool: # noqa: PLR0912, PLR09 raise Exception('ERROR: Please specify a WFP file to scan') if not os.path.exists(wfp_file) or not os.path.isfile(wfp_file): raise Exception(f'ERROR: Specified WFP file does not exist or is not a file: {wfp_file}') - cur_size = 0 queue_size = 0 file_count = 0 # count all files fingerprinted wfp_file_count = 0 # count number of files in each queue post scan_started = False - wfp = '' scan_block = '' + scan_size = 0 batch_context = None # Track SBOM context for the current batch - with open(wfp_file) as f: # Parse the WFP file - for line in f: - if line.startswith(WFP_FILE_START): - # First, add previous file's content to wfp before checking context - if scan_block: - wfp += scan_block - cur_size = len(wfp.encode('utf-8')) - file_path = Scanner._extract_file_path_from_line(line) - file_context = ( - self.scanoss_settings.get_sbom_context(file_path) - if self.scanoss_settings else SbomContext.empty() - ) + for file_path, wfp in self._iter_wfp_files(wfp_file): + file_count += 1 + wfp_size = len(wfp.encode('utf-8')) - # FLUSH: Context changed (different purls or scan_type) - if wfp and batch_context is not None and file_context != batch_context: - sbom = batch_context.to_payload() - self.threaded_scan.queue_add(wfp, sbom=sbom) - queue_size += 1 - wfp = '' - wfp_file_count = 0 - cur_size = 0 - batch_context = None + file_context = ( + self.scanoss_settings.get_sbom_context(file_path) + if self.scanoss_settings else SbomContext.empty() + ) - scan_block = line # Start storing the next file - batch_context = file_context - file_count += 1 - wfp_file_count += 1 - else: - scan_block += line # Store the rest of the WFP for this file - l_size = cur_size + len(scan_block.encode('utf-8')) - # FLUSH: Hit the max post size - if (wfp_file_count > self.post_file_count or l_size >= self.max_post_size) and wfp: - if self.debug and cur_size > self.max_post_size: - Scanner.print_stderr(f'Warning: Post size {cur_size} greater than limit {self.max_post_size}') - sbom = batch_context.to_payload() if batch_context else None - self.threaded_scan.queue_add(wfp, sbom=sbom) - queue_size += 1 - wfp = '' - wfp_file_count = 0 - batch_context = None - if not scan_started and queue_size > self.nb_threads: # Start scanning if we have something to do - scan_started = True - if not self.threaded_scan.run(wait=False): - self.print_stderr( - 'Warning: Some errors uncounted while scanning. Results might be incomplete.' - ) - success = False - # End for loop + # FLUSH: Context changed (different purls or scan_type) + if scan_block != '' and batch_context is not None and file_context != batch_context: + sbom = batch_context.to_payload() if batch_context else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + batch_context = None + + # FLUSH: Current file won't fit in batch (size limit) + if scan_block != '' and (wfp_size + scan_size) >= self.max_post_size: + sbom = batch_context.to_payload() if batch_context else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + batch_context = None + + # ADD current file to batch + scan_block += wfp + batch_context = file_context + scan_size = len(scan_block.encode('utf-8')) + wfp_file_count += 1 + + # FLUSH: Batch is full (file count or size limit) + if wfp_file_count > self.post_file_count or scan_size >= self.max_post_size: + sbom = batch_context.to_payload() if batch_context else None + self.threaded_scan.queue_add(scan_block, sbom=sbom) + queue_size += 1 + scan_block = '' + wfp_file_count = 0 + batch_context = None + + if not scan_started and queue_size > self.nb_threads: + scan_started = True + if not self.threaded_scan.run(wait=False): + self.print_stderr( + 'Warning: Some errors encountered while scanning. Results might be incomplete.' + ) + success = False + # End for loop if scan_block: - wfp += scan_block # Store the WFP for the current file - if wfp: sbom = batch_context.to_payload() if batch_context else None - self.threaded_scan.queue_add(wfp, sbom=sbom) + self.threaded_scan.queue_add(scan_block, sbom=sbom) queue_size += 1 if not self.__run_scan_threaded(scan_started, file_count): diff --git a/tests/test_iter_wfp_files.py b/tests/test_iter_wfp_files.py new file mode 100644 index 00000000..5f04be30 --- /dev/null +++ b/tests/test_iter_wfp_files.py @@ -0,0 +1,123 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2026, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import os +import shutil +import tempfile +import unittest + +from scanoss.scanner import Scanner + + +class TestIterWfpFiles(unittest.TestCase): + """Tests for Scanner._iter_wfp_files() static method.""" + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def _create_wfp_file(self, content: str) -> str: + path = os.path.join(self.tmp_dir, 'test.wfp') + with open(path, 'w') as f: + f.write(content) + return path + + def test_single_file_entry(self): + wfp_content = ( + 'file=abc123,100,src/main.c\n' + '4=abcdef\n' + '8=012345\n' + ) + wfp_file = self._create_wfp_file(wfp_content) + results = list(Scanner._iter_wfp_files(wfp_file)) + self.assertEqual(len(results), 1) + file_path, content = results[0] + self.assertEqual(file_path, 'src/main.c') + self.assertEqual(content, wfp_content) + + def test_multiple_file_entries(self): + wfp_content = ( + 'file=aaa,10,src/a.c\n' + '4=111111\n' + 'file=bbb,20,src/b.c\n' + '4=222222\n' + 'file=ccc,30,src/c.c\n' + '4=333333\n' + ) + wfp_file = self._create_wfp_file(wfp_content) + results = list(Scanner._iter_wfp_files(wfp_file)) + self.assertEqual(len(results), 3) + self.assertEqual(results[0][0], 'src/a.c') + self.assertEqual(results[1][0], 'src/b.c') + self.assertEqual(results[2][0], 'src/c.c') + + def test_content_preserved_exactly(self): + wfp_content = ( + 'file=aaa,10,src/a.c\n' + '4=111111\n' + '8=aaaaaa\n' + 'file=bbb,20,src/b.c\n' + '4=222222\n' + ) + wfp_file = self._create_wfp_file(wfp_content) + results = list(Scanner._iter_wfp_files(wfp_file)) + self.assertEqual(len(results), 2) + self.assertEqual(results[0][1], 'file=aaa,10,src/a.c\n4=111111\n8=aaaaaa\n') + self.assertEqual(results[1][1], 'file=bbb,20,src/b.c\n4=222222\n') + + def test_empty_file(self): + wfp_file = self._create_wfp_file('') + results = list(Scanner._iter_wfp_files(wfp_file)) + self.assertEqual(results, []) + + def test_file_entry_without_snippets(self): + wfp_content = 'file=abc123,100,src/only_header.c\n' + wfp_file = self._create_wfp_file(wfp_content) + results = list(Scanner._iter_wfp_files(wfp_file)) + self.assertEqual(len(results), 1) + file_path, content = results[0] + self.assertEqual(file_path, 'src/only_header.c') + self.assertEqual(content, 'file=abc123,100,src/only_header.c\n') + + def test_consecutive_file_entries(self): + wfp_content = ( + 'file=aaa,10,src/a.c\n' + 'file=bbb,20,src/b.c\n' + 'file=ccc,30,src/c.c\n' + ) + wfp_file = self._create_wfp_file(wfp_content) + results = list(Scanner._iter_wfp_files(wfp_file)) + self.assertEqual(len(results), 3) + self.assertEqual(results[0][0], 'src/a.c') + self.assertEqual(results[0][1], 'file=aaa,10,src/a.c\n') + self.assertEqual(results[1][0], 'src/b.c') + self.assertEqual(results[1][1], 'file=bbb,20,src/b.c\n') + self.assertEqual(results[2][0], 'src/c.c') + self.assertEqual(results[2][1], 'file=ccc,30,src/c.c\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_scan_wfp_file_threaded.py b/tests/test_scan_wfp_file_threaded.py new file mode 100644 index 00000000..794a708d --- /dev/null +++ b/tests/test_scan_wfp_file_threaded.py @@ -0,0 +1,185 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2026, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import os +import shutil +import tempfile +import unittest +from typing import NamedTuple, Optional +from unittest.mock import MagicMock, patch + +from scanoss.scanner import Scanner +from scanoss.scanoss_settings import SbomContext + + +class Batch(NamedTuple): + wfp: str + sbom: Optional[dict] + + +class TestScanWfpFileThreaded(unittest.TestCase): + """Tests for Scanner.scan_wfp_file_threaded() batching logic.""" + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def _create_wfp_file(self, content: str) -> str: + path = os.path.join(self.tmp_dir, 'test.wfp') + with open(path, 'w') as f: + f.write(content) + return path + + def _make_scanner(self, **overrides): + """Create a Scanner with __init__ bypassed and minimal attributes set.""" + with patch.object(Scanner, '__init__', lambda self: None): + scanner = Scanner() + scanner.scanoss_settings = None + scanner.threaded_scan = MagicMock() + scanner.max_post_size = 64 * 1024 + scanner.post_file_count = 32 + scanner.nb_threads = 5 + # Patch the private __run_scan_threaded to avoid thread infrastructure + scanner._Scanner__run_scan_threaded = MagicMock(return_value=True) + for key, value in overrides.items(): + setattr(scanner, key, value) + return scanner + + def _get_batches(self, scanner): + """Return list of Batch(wfp, sbom) from queue_add calls.""" + return [ + Batch(call.args[0], call.kwargs.get('sbom')) + for call in scanner.threaded_scan.queue_add.call_args_list + ] + + # ------------------------------------------------------------------ + # Test cases + # ------------------------------------------------------------------ + + def test_single_file_queued(self): + """A single file entry produces one queue_add call with the WFP content.""" + wfp = 'file=abc123,100,src/main.c\n4=aaaabbbb\n' + wfp_file = self._create_wfp_file(wfp) + scanner = self._make_scanner() + + result = scanner.scan_wfp_file_threaded(wfp_file) + + self.assertTrue(result) + batches = self._get_batches(scanner) + self.assertEqual(len(batches), 1) + self.assertEqual(batches[0].wfp, wfp) + self.assertIsNone(batches[0].sbom) + + def test_multiple_files_single_batch(self): + """Multiple small files that fit in one batch produce a single queue_add call.""" + wfp_lines = ( + 'file=aaa,10,src/a.c\n4=11112222\n' + 'file=bbb,20,src/b.c\n4=33334444\n' + 'file=ccc,30,src/c.c\n4=55556666\n' + ) + wfp_file = self._create_wfp_file(wfp_lines) + scanner = self._make_scanner() + + scanner.scan_wfp_file_threaded(wfp_file) + + batches = self._get_batches(scanner) + self.assertEqual(len(batches), 1) + # The batch should contain all three file entries concatenated + self.assertEqual(batches[0].wfp.count('file='), 3) + + def test_file_count_flush(self): + """When post_file_count is exceeded the batch is flushed. + + The flush condition is ``wfp_file_count > post_file_count`` (checked + *after* adding the current file). With ``post_file_count=1``: + - after file a: count=1, 1>1? no + - after file b: count=2, 2>1? yes → flush [a, b] + - after file c: count=1, 1>1? no → flushed at end-of-loop [c] + """ + wfp_lines = ( + 'file=aaa,10,src/a.c\n4=11112222\n' + 'file=bbb,20,src/b.c\n4=33334444\n' + 'file=ccc,30,src/c.c\n4=55556666\n' + ) + wfp_file = self._create_wfp_file(wfp_lines) + scanner = self._make_scanner(post_file_count=1) + + scanner.scan_wfp_file_threaded(wfp_file) + + batches = self._get_batches(scanner) + self.assertEqual(len(batches), 2) + # First batch: files a and b (flushed when count reaches 2 > 1) + self.assertEqual(batches[0].wfp.count('file='), 2) + # Second batch: file c (flushed at end of loop) + self.assertEqual(batches[1].wfp.count('file='), 1) + + def test_size_limit_flush(self): + """When accumulated WFP size exceeds max_post_size the batch is flushed before adding.""" + file_a = 'file=aaa,10,src/a.c\n4=11112222\n' + file_b = 'file=bbb,20,src/b.c\n4=33334444\n' + wfp_lines = file_a + file_b + wfp_file = self._create_wfp_file(wfp_lines) + + # Set max_post_size so file_a alone fits, but file_a + file_b would not. + # The pre-add size check: (wfp_size + scan_size) >= max_post_size + size_a = len(file_a.encode('utf-8')) + size_b = len(file_b.encode('utf-8')) + scanner = self._make_scanner(max_post_size=size_a + size_b - 1) + + scanner.scan_wfp_file_threaded(wfp_file) + + batches = self._get_batches(scanner) + self.assertEqual(len(batches), 2) + # First batch: file a (flushed because adding b would exceed limit) + self.assertIn('src/a.c', batches[0].wfp) + self.assertNotIn('src/b.c', batches[0].wfp) + # Second batch: file b (flushed at end of loop) + self.assertIn('src/b.c', batches[1].wfp) + + def test_empty_wfp_file(self): + """An empty WFP file results in no queue_add calls.""" + wfp_file = self._create_wfp_file('') + scanner = self._make_scanner() + + result = scanner.scan_wfp_file_threaded(wfp_file) + + self.assertTrue(result) + scanner.threaded_scan.queue_add.assert_not_called() + + def test_returns_true_on_success(self): + """Method returns True when __run_scan_threaded returns True.""" + wfp = 'file=abc123,100,src/main.c\n4=aaaabbbb\n' + wfp_file = self._create_wfp_file(wfp) + scanner = self._make_scanner() + + result = scanner.scan_wfp_file_threaded(wfp_file) + + self.assertTrue(result) + scanner._Scanner__run_scan_threaded.assert_called_once() + + +if __name__ == '__main__': + unittest.main() From f2baab295f0f8f8de8170bbead1d8c940caf8f6c Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 2 Mar 2026 16:55:41 +0100 Subject: [PATCH 450/489] refactor(scanner): add TODO for batching/flush logic deduplication in scan methods --- src/scanoss/scanner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 3df889b4..e4c64a29 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -493,6 +493,7 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 if self.scanoss_settings else SbomContext.empty() ) + # TODO: extract batching/flush logic into a shared helper to deduplicate scan_folder, scan_files, and scan_wfp_file_threaded # FLUSH: Context changed (different purls or scan_type) if scan_block != '' and batch_context is not None and file_context != batch_context: sbom = batch_context.to_payload() if batch_context else None From 66fe8bb34b64c6d77ace5d55c84ac0aa590eed6d Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 2 Mar 2026 18:00:49 +0100 Subject: [PATCH 451/489] refactor(settings): remove unused SbomContext.union() method --- src/scanoss/scanoss_settings.py | 14 ---------- tests/test_bom_path_matching.py | 45 --------------------------------- 2 files changed, 59 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 17702a9e..5b6335ba 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -132,20 +132,6 @@ def empty(cls) -> 'SbomContext': """Return empty context (no purls, no scan_type).""" return cls(purls=(), scan_type=None) - @classmethod - def union(cls, contexts: list) -> 'SbomContext': - """Merge multiple contexts: union of purls, first non-None scan_type wins.""" - all_purls = [] - scan_type = None - seen = set() - for ctx in contexts: - if scan_type is None and ctx.scan_type: - scan_type = ctx.scan_type - for p in ctx.purls: - if p not in seen: - seen.add(p) - all_purls.append(p) - return cls(purls=tuple(all_purls), scan_type=scan_type) def find_best_match(result_path: str, result_purls: List[str], entries: List[BomEntry]) -> Optional[BomEntry]: diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index 0c7f2fb3..a7e0ab0a 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -466,51 +466,6 @@ def test_sbom_context_empty(self): self.assertIsNone(context.scan_type) self.assertIsNone(context.to_payload()) - # -- SbomContext.union tests -- - - def test_sbom_context_union_empty_list(self): - """Union of empty list should return empty context""" - result = SbomContext.union([]) - self.assertEqual(result.purls, ()) - self.assertIsNone(result.scan_type) - - def test_sbom_context_union_single(self): - """Union of single context should return equivalent context""" - ctx = SbomContext(purls=('pkg:npm/a',), scan_type='identify') - result = SbomContext.union([ctx]) - self.assertEqual(result.purls, ('pkg:npm/a',)) - self.assertEqual(result.scan_type, 'identify') - - def test_sbom_context_union_multiple_same_type(self): - """Union of multiple contexts with same scan_type""" - ctx1 = SbomContext(purls=('pkg:npm/a',), scan_type='identify') - ctx2 = SbomContext(purls=('pkg:npm/b',), scan_type='identify') - result = SbomContext.union([ctx1, ctx2]) - self.assertEqual(result.purls, ('pkg:npm/a', 'pkg:npm/b')) - self.assertEqual(result.scan_type, 'identify') - - def test_sbom_context_union_mixed_types_first_wins(self): - """Union of mixed scan_types: first non-None wins""" - ctx1 = SbomContext(purls=('pkg:npm/a',), scan_type='identify') - ctx2 = SbomContext(purls=('pkg:npm/b',), scan_type='blacklist') - result = SbomContext.union([ctx1, ctx2]) - self.assertEqual(result.purls, ('pkg:npm/a', 'pkg:npm/b')) - self.assertEqual(result.scan_type, 'identify') - - def test_sbom_context_union_deduplicates_purls(self): - """Union should deduplicate purls while preserving order""" - ctx1 = SbomContext(purls=('pkg:npm/a', 'pkg:npm/b'), scan_type='identify') - ctx2 = SbomContext(purls=('pkg:npm/b', 'pkg:npm/c'), scan_type='identify') - result = SbomContext.union([ctx1, ctx2]) - self.assertEqual(result.purls, ('pkg:npm/a', 'pkg:npm/b', 'pkg:npm/c')) - - def test_sbom_context_union_skips_none_scan_type(self): - """Union should skip None scan_types and use first non-None""" - ctx1 = SbomContext(purls=('pkg:npm/a',), scan_type=None) - ctx2 = SbomContext(purls=('pkg:npm/b',), scan_type='blacklist') - result = SbomContext.union([ctx1, ctx2]) - self.assertEqual(result.scan_type, 'blacklist') - # -- Integration tests (get_sbom_context + to_payload) -- def test_folder_scoped_entry_included_when_matching(self): From b38de155b36c0cf6255dfc3dd7c76d84624be608 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 2 Mar 2026 19:06:59 +0100 Subject: [PATCH 452/489] feat(settings): send replace_with PURLs as identify context during scan --- src/scanoss/scanoss_settings.py | 47 +++++++++++++-- tests/test_bom_path_matching.py | 104 ++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 5b6335ba..2097d2ae 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -101,6 +101,8 @@ class ReplaceRule(BomEntry): @classmethod def from_dict(cls, data: dict) -> 'ReplaceRule': + if not data.get('replace_with'): + raise ValueError(f'Replace rule missing "replace_with" (purl: {data.get("purl")})') return cls( purl=data.get('purl'), path=data.get('path'), @@ -359,7 +361,13 @@ def get_bom_replace(self) -> List[ReplaceRule]: if self.settings_file_type == 'legacy': return [] raw = self._get_bom().get('replace', []) - return [ReplaceRule.from_dict(entry) for entry in raw] + rules = [] + for entry in raw: + try: + rules.append(ReplaceRule.from_dict(entry)) + except ValueError as e: + self.print_stderr(f'WARNING: {e}. Skipping.') + return rules def _get_purls_for_path(self, file_path: str, entries: List[BomEntry]) -> list: """ @@ -394,6 +402,33 @@ def _get_purls_for_path(self, file_path: str, entries: List[BomEntry]) -> list: # Sort by score descending (most specific first) return sorted(purl_scores.keys(), key=lambda p: -purl_scores[p]) + def _get_replace_with_purls_for_path(self, file_path: str, entries: List[ReplaceRule]) -> list: + """ + Extract replace_with purls from replace rules matching a given file path. + + At scan time we don't know the result PURLs yet, so only path matching + is used. Global rules (no path) always contribute their replace_with. + + Args: + file_path: File path to check + entries: List of ReplaceRule to check against + + Returns: + List of replace_with purl strings ordered by specificity (most specific first) + """ + if not entries: + return [] + + purl_scores = {} + for entry in entries: + if entry.matches_path(file_path): + entry_path = entry.path or '' + score = entry.priority + len(entry_path) + if entry.replace_with not in purl_scores or score > purl_scores[entry.replace_with]: + purl_scores[entry.replace_with] = score + + return sorted(purl_scores.keys(), key=lambda p: -purl_scores[p]) + def get_sbom_context(self, file_path: str) -> SbomContext: """ Get SBOM context matching a file path. @@ -420,10 +455,14 @@ def get_sbom_context(self, file_path: str) -> SbomContext: return SbomContext(purls=tuple(purls), scan_type=self.scan_type) return SbomContext.empty() - # New format: try include first, then exclude + # Collect include + replace_with purls, fall back to exclude include_purls = self._get_purls_for_path(file_path, self.get_bom_include()) - if include_purls: - return SbomContext(purls=tuple(include_purls), scan_type='identify') + replace_purls = self._get_replace_with_purls_for_path(file_path, self.get_bom_replace()) + + # Merge include + replace_with, deduplicating (include order first) + all_identify = list(dict.fromkeys(include_purls + replace_purls)) + if all_identify: + return SbomContext(purls=tuple(all_identify), scan_type='identify') exclude_purls = self._get_purls_for_path(file_path, self.get_bom_exclude()) if exclude_purls: diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index a7e0ab0a..bc2944eb 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -533,6 +533,110 @@ def test_include_no_match_falls_back_to_exclude(self): self.assertEqual(context_other.purls, ('pkg:npm/lodash',)) self.assertEqual(context_other.scan_type, 'blacklist') + # -- replace_with as identify context tests -- + + def test_replace_with_sent_as_identify_context(self): + """Replace rule with matching path sends replace_with PURL as identify context""" + settings = self._make_settings({ + 'bom': { + 'replace': [ + {'path': 'vendor/', 'purl': 'pkg:npm/old-lib', 'replace_with': 'pkg:npm/new-lib'}, + ], + } + }) + context = settings.get_sbom_context('vendor/file.js') + self.assertEqual(context.purls, ('pkg:npm/new-lib',)) + self.assertEqual(context.scan_type, 'identify') + + def test_replace_with_global_sent_for_all_files(self): + """Replace rule without path sends replace_with PURL for any file""" + settings = self._make_settings({ + 'bom': { + 'replace': [ + {'purl': 'pkg:npm/old-lib', 'replace_with': 'pkg:npm/new-lib'}, + ], + } + }) + context_a = settings.get_sbom_context('src/app.js') + self.assertEqual(context_a.purls, ('pkg:npm/new-lib',)) + self.assertEqual(context_a.scan_type, 'identify') + + context_b = settings.get_sbom_context('lib/utils.c') + self.assertEqual(context_b.purls, ('pkg:npm/new-lib',)) + self.assertEqual(context_b.scan_type, 'identify') + + def test_replace_with_merged_with_include(self): + """Both include and replace_with PURLs appear in identify context, deduplicated""" + settings = self._make_settings({ + 'bom': { + 'include': [ + {'purl': 'pkg:npm/react'}, + ], + 'replace': [ + {'purl': 'pkg:npm/old-lib', 'replace_with': 'pkg:npm/new-lib'}, + {'purl': 'pkg:npm/dup', 'replace_with': 'pkg:npm/react'}, # duplicate of include + ], + } + }) + context = settings.get_sbom_context('src/app.js') + self.assertEqual(context.scan_type, 'identify') + self.assertIn('pkg:npm/react', context.purls) + self.assertIn('pkg:npm/new-lib', context.purls) + # Deduplicated: react should appear only once + self.assertEqual(context.purls.count('pkg:npm/react'), 1) + + def test_replace_with_no_match_falls_to_exclude(self): + """Path-scoped replace that doesn't match should not interfere with exclude fallback""" + settings = self._make_settings({ + 'bom': { + 'replace': [ + {'path': 'vendor/', 'purl': 'pkg:npm/old-lib', 'replace_with': 'pkg:npm/new-lib'}, + ], + 'exclude': [ + {'purl': 'pkg:npm/lodash'}, + ], + } + }) + context = settings.get_sbom_context('src/app.js') + self.assertEqual(context.purls, ('pkg:npm/lodash',)) + self.assertEqual(context.scan_type, 'blacklist') + + def test_replace_with_overrides_exclude(self): + """When both replace and exclude match, identify context wins""" + settings = self._make_settings({ + 'bom': { + 'replace': [ + {'purl': 'pkg:npm/old-lib', 'replace_with': 'pkg:npm/new-lib'}, + ], + 'exclude': [ + {'purl': 'pkg:npm/lodash'}, + ], + } + }) + context = settings.get_sbom_context('src/app.js') + self.assertEqual(context.purls, ('pkg:npm/new-lib',)) + self.assertEqual(context.scan_type, 'identify') + + def test_replace_missing_replace_with_warns_and_skips(self): + """Replace rule missing replace_with should be skipped with a warning""" + settings = self._make_settings({ + 'bom': { + 'replace': [ + {'purl': 'pkg:npm/bad-rule'}, # missing replace_with + {'purl': 'pkg:npm/old-lib', 'replace_with': 'pkg:npm/new-lib'}, + ], + } + }) + from unittest.mock import patch + with patch.object(settings, 'print_stderr') as mock_stderr: + rules = settings.get_bom_replace() + # Invalid entry filtered out + self.assertEqual(len(rules), 1) + self.assertEqual(rules[0].replace_with, 'pkg:npm/new-lib') + # Warning was printed + mock_stderr.assert_called_once() + self.assertIn('replace_with', mock_stderr.call_args[0][0]) + class TestScannerSbomPayload(unittest.TestCase): """End-to-end tests: verify Scanner sends the correct SBOM payload in HTTP POST requests""" From 5378ff272c32e72fe963348d6e0099bbc6768c31 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 3 Mar 2026 10:33:50 +0100 Subject: [PATCH 453/489] feat(settings): make BOM path matching agnostic to trailing slashes --- src/scanoss/scanoss_settings.py | 14 ++++---- src/scanoss/scanpostprocessor.py | 4 ++- tests/test_bom_path_matching.py | 59 ++++++++++++++++++++++++++++---- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/scanoss/scanoss_settings.py b/src/scanoss/scanoss_settings.py index 2097d2ae..7a39fbaa 100644 --- a/src/scanoss/scanoss_settings.py +++ b/src/scanoss/scanoss_settings.py @@ -48,16 +48,17 @@ class BomEntry: @classmethod def from_dict(cls, data: dict) -> 'BomEntry': + raw_path = data.get('path') + path = raw_path.rstrip('/') if raw_path else None return cls( purl=data.get('purl'), - path=data.get('path'), + path=path, comment=data.get('comment'), ) def matches_path(self, result_path: str) -> bool: """ Check if this entry's path matches a result path. - Folder paths (ending with '/') use prefix matching; file paths use exact matching. Args: result_path: Path from the scan result @@ -67,9 +68,8 @@ def matches_path(self, result_path: str) -> bool: """ if not self.path: return True - if self.path.endswith('/'): - return result_path.startswith(self.path) - return self.path == result_path + path = self.path.rstrip('/') + return path == result_path or result_path.startswith(path + '/') @property def priority(self) -> int: @@ -103,9 +103,11 @@ class ReplaceRule(BomEntry): def from_dict(cls, data: dict) -> 'ReplaceRule': if not data.get('replace_with'): raise ValueError(f'Replace rule missing "replace_with" (purl: {data.get("purl")})') + raw_path = data.get('path') + path = raw_path.rstrip('/') if raw_path else None return cls( purl=data.get('purl'), - path=data.get('path'), + path=path, comment=data.get('comment'), replace_with=data.get('replace_with'), license=data.get('license'), diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index b2f927c8..6f47276a 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -45,7 +45,9 @@ def _get_match_type_message(result_path: str, bom_entry: BomEntry, action: str) """ entry_path = bom_entry.path or '' if entry_path and bom_entry.purl: - match_kind = 'folder' if entry_path.endswith('/') else 'file' + # Result keys are always file paths, so exact match means file-level rule; + # otherwise the match came via folder prefix. + match_kind = 'file' if entry_path == result_path else 'folder' message = f"{action} '{result_path}'. Full match found ({match_kind} + purl)." elif bom_entry.purl: message = f"{action} '{result_path}'. Found PURL match." diff --git a/tests/test_bom_path_matching.py b/tests/test_bom_path_matching.py index bc2944eb..ae0db00c 100644 --- a/tests/test_bom_path_matching.py +++ b/tests/test_bom_path_matching.py @@ -67,11 +67,56 @@ def test_folder_root_prefix(self): self.assertTrue(BomEntry(path='src/').matches_path('src/main.c')) self.assertTrue(BomEntry(path='src/').matches_path('src/vendor/deep/file.c')) + def test_folder_without_trailing_slash(self): + """Paths without trailing slash should still do prefix matching""" + self.assertTrue(BomEntry(path='src/vendor').matches_path('src/vendor/lib.c')) + self.assertTrue(BomEntry(path='src/vendor').matches_path('src/vendor/sub/deep.c')) + + def test_folder_without_trailing_slash_no_match(self): + self.assertFalse(BomEntry(path='src/vendor').matches_path('src/other/lib.c')) + self.assertFalse(BomEntry(path='src/vendor').matches_path('src/vendorlib.c')) + + def test_folder_without_trailing_slash_exact_match(self): + """Path without trailing slash should still match the exact path""" + self.assertTrue(BomEntry(path='src/vendor').matches_path('src/vendor')) + def test_exact_path_does_not_prefix_match(self): - """File paths (no trailing slash) should not do prefix matching""" + """File paths should not do prefix matching on partial names""" self.assertFalse(BomEntry(path='src/main.c').matches_path('src/main.cpp')) +class TestFromDictNormalization(unittest.TestCase): + """Unit tests for trailing-slash normalization in from_dict""" + + def test_bom_entry_strips_trailing_slash(self): + entry = BomEntry.from_dict({'path': 'src/vendor/'}) + self.assertEqual(entry.path, 'src/vendor') + + def test_bom_entry_no_trailing_slash_unchanged(self): + entry = BomEntry.from_dict({'path': 'src/main.c'}) + self.assertEqual(entry.path, 'src/main.c') + + def test_bom_entry_none_path(self): + entry = BomEntry.from_dict({}) + self.assertIsNone(entry.path) + + def test_replace_rule_strips_trailing_slash(self): + entry = ReplaceRule.from_dict({ + 'path': 'src/vendor/', + 'purl': 'pkg:npm/old', + 'replace_with': 'pkg:npm/new', + }) + self.assertEqual(entry.path, 'src/vendor') + + def test_replace_rule_no_trailing_slash_unchanged(self): + entry = ReplaceRule.from_dict({ + 'path': 'src/main.c', + 'purl': 'pkg:npm/old', + 'replace_with': 'pkg:npm/new', + }) + self.assertEqual(entry.path, 'src/main.c') + + class TestEntryPriority(unittest.TestCase): """Unit tests for the BomEntry.priority property""" @@ -397,13 +442,13 @@ def test_get_sbom_context_ordered_by_specificity(self): 'bom': { 'include': [ {'purl': 'pkg:npm/global'}, # score: 2 (purl only) - {'path': 'src/', 'purl': 'pkg:npm/src-lib'}, # score: 4 + 4 = 8 - {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, # score: 4 + 11 = 15 + {'path': 'src/', 'purl': 'pkg:npm/src-lib'}, # score: 4 + 3 = 7 (normalized to 'src') + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor-lib'}, # score: 4 + 10 = 14 (normalized to 'src/vendor') ], } }) context = settings.get_sbom_context('src/vendor/lib.c') - # Most specific first: vendor-lib (15), src-lib (8), global (2) + # Most specific first: vendor-lib (14), src-lib (7), global (2) self.assertEqual(context.purls, ('pkg:npm/vendor-lib', 'pkg:npm/src-lib', 'pkg:npm/global')) def test_get_sbom_context_file_path_most_specific(self): @@ -911,8 +956,8 @@ def test_payload_preserves_specificity_order(self): 'bom': { 'include': [ {'purl': 'pkg:npm/global'}, # score: 2 - {'path': 'src/', 'purl': 'pkg:npm/src-lib'}, # score: 4 + 4 = 8 - {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor'}, # score: 4 + 11 = 15 + {'path': 'src/', 'purl': 'pkg:npm/src-lib'}, # score: 4 + 3 = 7 (normalized to 'src') + {'path': 'src/vendor/', 'purl': 'pkg:npm/vendor'}, # score: 4 + 10 = 14 (normalized to 'src/vendor') ], } }) @@ -963,7 +1008,7 @@ def test_nested_folder_deeper_wins(self): self.assertEqual(len(payloads), 1) assets = json.loads(payloads[0]['assets']) purl_order = [c['purl'] for c in assets['components']] - # deep (score 15) before shallow (score 8) + # deep (score 14) before shallow (score 7) self.assertEqual(purl_order[0], 'pkg:npm/deep') def test_file_path_beats_folder_path_ordering(self): From b71bff734e5a5788d8ec217aa08a36eaabd18d17 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 3 Mar 2026 13:59:48 +0100 Subject: [PATCH 454/489] style(scanner): reformat TODO comment for better readability --- src/scanoss/scanner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index e4c64a29..7ccf37ba 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -493,7 +493,8 @@ def scan_folder(self, scan_dir: str) -> bool: # noqa: PLR0912, PLR0915 if self.scanoss_settings else SbomContext.empty() ) - # TODO: extract batching/flush logic into a shared helper to deduplicate scan_folder, scan_files, and scan_wfp_file_threaded + # TODO: extract batching/flush logic into a shared helper to deduplicate + # scan_folder, scan_files, and scan_wfp_file_threaded # FLUSH: Context changed (different purls or scan_type) if scan_block != '' and batch_context is not None and file_context != batch_context: sbom = batch_context.to_payload() if batch_context else None From cd4d02c144fcf1012b412566629ef4c335408ffd Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 3 Mar 2026 15:48:50 +0100 Subject: [PATCH 455/489] chore(CHANGELOG): update with folder-level BOM filtering, SBOM handling refactors, and path matching improvements --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b6d6e3e..dc25d23e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -- Upcoming changes... +### Added +- Added folder-level (path-scoped) BOM filtering for `include`, `exclude`, `remove`, and `replace` rules + - BOM rules can now target specific folders (e.g., `"path": "src/vendor/"`) in addition to individual files + - Priority-based matching: path+purl (highest) > purl-only > path-only (lowest); longer paths win ties + - Path-scoped `include` entries are sent as identify context only for files under the matching folder + - Path-scoped `exclude` entries are sent as blacklist context only for files under the matching folder +- Replace rules now contribute their `replace_with` PURL to the scan context, improving server-side matching +- BOM path matching is agnostic to trailing slashes (`src/vendor/` and `src/vendor` are equivalent) + +### Changed +- Refactored SBOM handling: replaced global SBOM context with per-file `SbomContext` resolution via `ScanossSettings.get_sbom_context()` +- Refactored `BomEntry` from TypedDict to dataclass hierarchy with `matches_path()`, `matches_purl()`, and `priority` support +- Extracted `_iter_wfp_files` generator to simplify WFP parsing in `scan_wfp_file_threaded` ## [1.45.1] - 2026-02-23 ### Fixed From 861f06b0d1b717d9d7a4d357b85d9d701c13b77d Mon Sep 17 00:00:00 2001 From: eeisegn Date: Wed, 4 Mar 2026 08:54:11 +0000 Subject: [PATCH 456/489] added extra debug --- CHANGELOG.md | 5 ++++- src/scanoss/__init__.py | 2 +- src/scanoss/scanpostprocessor.py | 31 ++++++++++++++++++++----------- src/scanoss/threadedscanning.py | 3 +++ 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc25d23e..92227ca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [1.46.0] - 2026-03-04 ### Added - Added folder-level (path-scoped) BOM filtering for `include`, `exclude`, `remove`, and `replace` rules - BOM rules can now target specific folders (e.g., `"path": "src/vendor/"`) in addition to individual files @@ -812,4 +814,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.43.1]: https://github.com/scanoss/scanoss.py/compare/v1.43.0...v1.43.1 [1.44.0]: https://github.com/scanoss/scanoss.py/compare/v1.43.1...v1.44.0 [1.45.0]: https://github.com/scanoss/scanoss.py/compare/v1.44.0...v1.45.0 -[1.45.1]: https://github.com/scanoss/scanoss.py/compare/v1.45.0...v1.45.1 \ No newline at end of file +[1.45.1]: https://github.com/scanoss/scanoss.py/compare/v1.45.0...v1.45.1 +[1.46.0]: https://github.com/scanoss/scanoss.py/compare/v1.45.1...v1.46.0 \ No newline at end of file diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 8a47bde0..cfb554be 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.45.1' +__version__ = '1.46.0' diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 6f47276a..f06def20 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -211,7 +211,8 @@ def _should_replace_result( result_purls = result.get('purl', []) match = find_best_match(result_path, result_purls, to_replace_entries) if match and isinstance(match, ReplaceRule) and match.replace_with: - self._print_message(result_path, result_purls, match, 'Replacing') + if self.debug: + self._print_message(result_path, result_purls, match, 'Replacing') return True, match.replace_with return False, None @@ -232,7 +233,8 @@ def _should_remove_result(self, result_path: str, result: dict, to_remove_entrie result_purls = result.get('purl', []) match = find_best_match(result_path, result_purls, to_remove_entries) if match: - self._print_message(result_path, result_purls, match, 'Removing') + if self.debug: + self._print_message(result_path, result_purls, match, 'Removing') return True return False @@ -246,17 +248,24 @@ def _print_message(self, result_path: str, result_purls: List[str], bom_entry: B bom_entry (BomEntry): Matched BOM entry action (str): Action being performed """ - message = ( - f'{_get_match_type_message(result_path, bom_entry, action)} \n' - f'Details:\n' - f' - PURLs: {", ".join(result_purls)}\n' - f" - Path: '{result_path}'\n" - ) + if not self.debug: + return if action == 'Replacing' and isinstance(bom_entry, ReplaceRule): - message += f" - {action} with '{bom_entry.replace_with}'" + message = ( + f'{_get_match_type_message(result_path, bom_entry, action)}\n' + f'Details:\n' + f' - PURLs: {", ".join(result_purls)}\n' + f' - Replace with: {bom_entry.replace_with}\n' + f" - Path: '{result_path}'" + ) + else: + message = ( + f'{_get_match_type_message(result_path, bom_entry, action)}\n' + f'Details:\n' + f' - PURLs: {", ".join(result_purls)}\n' + f" - Path: '{result_path}'" + ) self.print_debug(message) - - # # End of ScanPostProcessor Class # diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index 97515104..5e772bf3 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -144,6 +144,9 @@ def queue_add(self, wfp: str, sbom: dict = None) -> None: :param wfp: WFP to add to queue :param sbom: Per-request SBOM context (optional, overrides global SBOM) """ + if sbom and self.debug: + w = (wfp.split('\n'))[0] # show the first file to help debug context + self.print_debug(f'Adding SBOM context: {sbom} to {w} ...') if wfp is None or wfp == '': self.print_stderr('Warning: empty WFP. Skipping from scan...') else: From af81ceffd709fffacfe7247e2eb4ceb828a47325 Mon Sep 17 00:00:00 2001 From: eeisegn Date: Wed, 4 Mar 2026 10:13:45 +0000 Subject: [PATCH 457/489] limit the number of lines split --- src/scanoss/threadedscanning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/threadedscanning.py b/src/scanoss/threadedscanning.py index 5e772bf3..596db599 100644 --- a/src/scanoss/threadedscanning.py +++ b/src/scanoss/threadedscanning.py @@ -145,7 +145,7 @@ def queue_add(self, wfp: str, sbom: dict = None) -> None: :param sbom: Per-request SBOM context (optional, overrides global SBOM) """ if sbom and self.debug: - w = (wfp.split('\n'))[0] # show the first file to help debug context + w = (wfp.split('\n', maxsplit=1))[0] # show the first file to help debug context. self.print_debug(f'Adding SBOM context: {sbom} to {w} ...') if wfp is None or wfp == '': self.print_stderr('Warning: empty WFP. Skipping from scan...') From f18036afdef94827e5d38d73f7c613f92a608e5a Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Mar 2026 15:40:42 +0100 Subject: [PATCH 458/489] feat(dependencies): add skip patterns for dependency filtering --- CHANGELOG.md | 2 + src/scanoss/cli.py | 18 +- src/scanoss/data/scanoss-settings-schema.json | 13 ++ src/scanoss/scancodedeps.py | 32 +++- src/scanoss/scanner.py | 9 +- src/scanoss/threadeddependencies.py | 6 +- tests/test_dependency_skip_patterns.py | 169 ++++++++++++++++++ 7 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 tests/test_dependency_skip_patterns.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 92227ca2..5f03f7f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Added support for skipping dependency files, configurable via `settings.skip.patterns.dependencies` ## [1.46.0] - 2026-03-04 ### Added diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 8216d85e..e31d224b 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1097,7 +1097,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 ) # Scanoss settings options - for p in [p_folder_scan, p_scan, p_wfp, p_folder_hash]: + for p in [p_folder_scan, p_scan, p_wfp, p_folder_hash, p_dep]: p.add_argument( '--settings', '-st', @@ -1730,10 +1730,22 @@ def dependency(parser, args): if args.output: initialise_empty_file(args.output) + scanoss_settings = None + if not args.skip_settings_file: + scanoss_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) + try: + scanoss_settings.load_json_file(args.settings, args.scan_loc) + except ScanossSettingsError as e: + print_stderr(f'Error: {e}') + sys.exit(1) + sc_deps = ScancodeDeps( - debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout + debug=args.debug, quiet=args.quiet, trace=args.trace, sc_command=args.sc_command, timeout=args.sc_timeout, + scanoss_settings=scanoss_settings, ) - if not sc_deps.get_dependencies(what_to_scan=args.scan_loc, result_output=args.output): + if not sc_deps.get_dependencies( + what_to_scan=args.scan_loc, result_output=args.output + ): sys.exit(1) return None diff --git a/src/scanoss/data/scanoss-settings-schema.json b/src/scanoss/data/scanoss-settings-schema.json index c103914d..85c551ab 100644 --- a/src/scanoss/data/scanoss-settings-schema.json +++ b/src/scanoss/data/scanoss-settings-schema.json @@ -65,6 +65,19 @@ ] }, "uniqueItems": true + }, + "dependencies": { + "type": "array", + "description": "List of glob patterns to skip dependency files from dependency analysis", + "items": { + "type": "string", + "examples": [ + "vendor/**", + "third_party/", + "node_modules/**" + ] + }, + "uniqueItems": true } } }, diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index 677c7774..f920d3f5 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -26,6 +26,8 @@ import os.path import subprocess +from pathspec import GitIgnoreSpec + from .scanossbase import ScanossBase @@ -43,6 +45,7 @@ def __init__( scan_output: str = None, timeout: int = 600, sc_command: str = None, + scanoss_settings=None, ): """ Initialise ScancodeDeps class @@ -55,6 +58,7 @@ def __init__( self.scan_output = scan_output self.sc_command = sc_command if sc_command else 'scancode' self.output_file = output_file if output_file else 'scancode-dependencies.json' + self.scanoss_settings = scanoss_settings def __log_result(self, string, outfile=None): """ @@ -183,7 +187,31 @@ def produce_from_str(self, json_str: str) -> dict: return None return self.produce_from_json(data) - def get_dependencies(self, output_file: str = None, what_to_scan: str = None, result_output: str = None) -> bool: + def filter_dependencies_by_path(self, deps: dict) -> dict: + """Filter dependency files by path using skip patterns from scanoss_settings. + + :param deps: dependency dict with 'files' key + :return: filtered dependency dict + """ + if not self.scanoss_settings: + return deps + patterns = self.scanoss_settings.get_skip_patterns('dependencies') + if not patterns: + return deps + spec = GitIgnoreSpec.from_lines(patterns) + all_files = deps.get('files', []) + filtered_files = [] + for f in all_files: + file_path = f.get('file', '') + if spec.match_file(file_path): + self.print_debug(f'Skipping dependency file: {file_path} (matches skip pattern)') + else: + filtered_files.append(f) + return {'files': filtered_files} + + def get_dependencies( + self, output_file: str = None, what_to_scan: str = None, result_output: str = None + ) -> bool: """ Get the dependencies for the required file/directory and output the JSON results :param output_file: temporary scanocde file to write interim results to @@ -196,6 +224,8 @@ def get_dependencies(self, output_file: str = None, what_to_scan: str = None, re return False self.print_msg('Producing summary...') deps = self.produce_from_file(output_file) + if deps: + deps = self.filter_dependencies_by_path(deps) deps = self.__remove_dep_scope(deps) self.remove_interim_file(output_file) if not deps: diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 7ccf37ba..4140dbff 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -202,7 +202,10 @@ def __init__( # noqa: PLR0913, PLR0915 ranking=scan_settings.ranking, ranking_threshold=scan_settings.ranking_threshold, ) - sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command) + sc_deps = ScancodeDeps( + debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command, + scanoss_settings=scanoss_settings, + ) grpc_api = ScanossGrpc( url=grpc_url, debug=debug, @@ -218,7 +221,9 @@ def __init__( # noqa: PLR0913, PLR0915 ignore_cert_errors=ignore_cert_errors, use_grpc=use_grpc ) - self.threaded_deps = ThreadedDependencies(sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace) + self.threaded_deps = ThreadedDependencies( + sc_deps, grpc_api, debug=debug, quiet=quiet, trace=trace + ) self.nb_threads = nb_threads if nb_threads and nb_threads > 0: self.threaded_scan = ThreadedScanning( diff --git a/src/scanoss/threadeddependencies.py b/src/scanoss/threadeddependencies.py index c083f4f4..9fcd4395 100644 --- a/src/scanoss/threadeddependencies.py +++ b/src/scanoss/threadeddependencies.py @@ -63,7 +63,7 @@ class ThreadedDependencies(ScanossBase): inputs: queue.Queue = queue.Queue() output: queue.Queue = queue.Queue() - def __init__( # noqa: PLR0913 + def __init__( self, sc_deps: ScancodeDeps, grpc_api: ScanossGrpc, @@ -196,10 +196,14 @@ def scan_dependencies( # noqa: PLR0912 deps = None if what_to_scan.startswith(DEP_FILE_PREFIX): # We have a pre-parsed dependency file, load it deps = self.sc_deps.load_from_file(what_to_scan.strip(DEP_FILE_PREFIX)) + if deps: + deps = self.sc_deps.filter_dependencies_by_path(deps) elif not self.sc_deps.run_scan(what_to_scan=what_to_scan): self._errors = True else: deps = self.sc_deps.produce_from_file() + if deps: + deps = self.sc_deps.filter_dependencies_by_path(deps) if dep_scope is not None: self.print_debug(f'Filtering {dep_scope.name} dependencies') if dep_scope_include is not None: diff --git a/tests/test_dependency_skip_patterns.py b/tests/test_dependency_skip_patterns.py new file mode 100644 index 00000000..7f4eb2ce --- /dev/null +++ b/tests/test_dependency_skip_patterns.py @@ -0,0 +1,169 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2026, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import unittest + +from scanoss.scancodedeps import ScancodeDeps +from scanoss.scanoss_settings import ScanossSettings + + +SAMPLE_DEPS = { + 'files': [ + {'file': 'package.json', 'purls': [{'purl': 'pkg:npm/express@4.18.0'}]}, + {'file': 'vendor/package.json', 'purls': [{'purl': 'pkg:npm/lodash@4.17.21'}]}, + {'file': 'vendor/sub/package.json', 'purls': [{'purl': 'pkg:npm/axios@1.0.0'}]}, + {'file': 'third_party/lib/requirements.txt', 'purls': [{'purl': 'pkg:pypi/requests@2.28.0'}]}, + {'file': 'src/go.mod', 'purls': [{'purl': 'pkg:golang/github.com/gin-gonic/gin@1.9.0'}]}, + ] +} + + +def _make_settings(patterns): + """Helper to create a ScanossSettings with dependency skip patterns.""" + settings = ScanossSettings(debug=True) + settings.data = { + 'settings': { + 'skip': { + 'patterns': { + 'dependencies': patterns, + } + } + } + } + return settings + + +def _make_sc_deps(scanoss_settings=None): + """Helper to create a ScancodeDeps with optional settings.""" + return ScancodeDeps(debug=True, scanoss_settings=scanoss_settings) + + + +class TestDependencySkipPatterns(unittest.TestCase): + """Tests for dependency skip patterns filtering on ScancodeDeps.""" + + def test_no_settings_no_filtering(self): + """No settings -> deps returned unchanged.""" + sc = _make_sc_deps(scanoss_settings=None) + result = sc.filter_dependencies_by_path(SAMPLE_DEPS) + self.assertEqual(result, SAMPLE_DEPS) + + def test_empty_patterns_no_filtering(self): + """Empty patterns list -> deps returned unchanged.""" + settings = _make_settings([]) + sc = _make_sc_deps(scanoss_settings=settings) + result = sc.filter_dependencies_by_path(SAMPLE_DEPS) + self.assertEqual(result, SAMPLE_DEPS) + + def test_exact_path_match(self): + """Exact path match -> that file is skipped.""" + settings = _make_settings(['vendor/package.json']) + sc = _make_sc_deps(scanoss_settings=settings) + result = sc.filter_dependencies_by_path(SAMPLE_DEPS) + files = result['files'] + matched_paths = [f['file'] for f in files] + self.assertNotIn('vendor/package.json', matched_paths) + self.assertIn('package.json', matched_paths) + self.assertIn('src/go.mod', matched_paths) + + def test_glob_pattern_vendor(self): + """Glob pattern vendor/** -> all files under vendor/ skipped.""" + settings = _make_settings(['vendor/**']) + sc = _make_sc_deps(scanoss_settings=settings) + result = sc.filter_dependencies_by_path(SAMPLE_DEPS) + files = result['files'] + matched_paths = [f['file'] for f in files] + self.assertNotIn('vendor/package.json', matched_paths) + self.assertNotIn('vendor/sub/package.json', matched_paths) + self.assertIn('package.json', matched_paths) + self.assertIn('third_party/lib/requirements.txt', matched_paths) + self.assertIn('src/go.mod', matched_paths) + + def test_directory_pattern(self): + """Directory pattern third_party/ -> files under it skipped.""" + settings = _make_settings(['third_party/']) + sc = _make_sc_deps(scanoss_settings=settings) + result = sc.filter_dependencies_by_path(SAMPLE_DEPS) + files = result['files'] + matched_paths = [f['file'] for f in files] + self.assertNotIn('third_party/lib/requirements.txt', matched_paths) + self.assertIn('package.json', matched_paths) + self.assertIn('vendor/package.json', matched_paths) + + def test_multiple_patterns(self): + """Multiple patterns -> all matching files skipped.""" + settings = _make_settings(['vendor/**', 'third_party/']) + sc = _make_sc_deps(scanoss_settings=settings) + result = sc.filter_dependencies_by_path(SAMPLE_DEPS) + files = result['files'] + matched_paths = [f['file'] for f in files] + self.assertNotIn('vendor/package.json', matched_paths) + self.assertNotIn('vendor/sub/package.json', matched_paths) + self.assertNotIn('third_party/lib/requirements.txt', matched_paths) + self.assertIn('package.json', matched_paths) + self.assertIn('src/go.mod', matched_paths) + self.assertEqual(len(files), 2) + + def test_no_match_all_kept(self): + """Pattern that matches nothing -> all files kept.""" + settings = _make_settings(['nonexistent/**']) + sc = _make_sc_deps(scanoss_settings=settings) + result = sc.filter_dependencies_by_path(SAMPLE_DEPS) + self.assertEqual(len(result['files']), len(SAMPLE_DEPS['files'])) + + +class TestGetSkipPatternsDependencies(unittest.TestCase): + """Tests for ScanossSettings.get_skip_patterns('dependencies').""" + + def test_returns_correct_data(self): + """get_skip_patterns('dependencies') returns the configured patterns.""" + settings = _make_settings(['vendor/**', 'third_party/']) + result = settings.get_skip_patterns('dependencies') + self.assertEqual(result, ['vendor/**', 'third_party/']) + + def test_returns_empty_when_key_missing(self): + """get_skip_patterns('dependencies') returns [] when key is absent (backward compat).""" + settings = ScanossSettings(debug=True) + settings.data = { + 'settings': { + 'skip': { + 'patterns': { + 'scanning': ['*.log'], + } + } + } + } + result = settings.get_skip_patterns('dependencies') + self.assertEqual(result, []) + + def test_returns_empty_when_no_settings(self): + """get_skip_patterns('dependencies') returns [] with empty data.""" + settings = ScanossSettings(debug=True) + settings.data = {} + result = settings.get_skip_patterns('dependencies') + self.assertEqual(result, []) + + +if __name__ == '__main__': + unittest.main() From 77c1ff6333a7e7a8bf9dbb31209176af22097699 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Mar 2026 16:00:14 +0100 Subject: [PATCH 459/489] fix(cli): reject conflicting --settings and --skip-settings-file in dep command --- src/scanoss/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index e31d224b..8905c341 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1730,6 +1730,9 @@ def dependency(parser, args): if args.output: initialise_empty_file(args.output) + if args.settings and args.skip_settings_file: + print_stderr('ERROR: Cannot specify both --settings and --skip-file-settings options.') + sys.exit(1) scanoss_settings = None if not args.skip_settings_file: scanoss_settings = ScanossSettings(debug=args.debug, trace=args.trace, quiet=args.quiet) From 85e97c6a6e04e4dad4d918c48f8a2b28d0bbc8ef Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Mar 2026 16:02:47 +0100 Subject: [PATCH 460/489] fix(lint): resolve pre-existing lint errors in scancodedeps.py --- src/scanoss/scancodedeps.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index f920d3f5..267b1a5e 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -36,7 +36,7 @@ class ScancodeDeps(ScanossBase): SCANOSS dependency scanning class """ - def __init__( + def __init__( # noqa: PLR0913 self, debug: bool = False, quiet: bool = False, @@ -81,12 +81,12 @@ def remove_interim_file(self, output_file: str = None): output_file = self.output_file if os.path.isfile(output_file): try: - self.print_trace(f'Cleaning temporary scancode files...') + self.print_trace('Cleaning temporary scancode files...') os.remove(output_file) except Exception as e: self.print_stderr(f'Warning: Failed to remove temporary file {output_file}: {e}') - def produce_from_json(self, data: json) -> dict: + def produce_from_json(self, data: json) -> dict: # noqa: PLR0912 """ Parse the given input JSON string and return Dependency summary :param data: json - JSON object @@ -95,7 +95,7 @@ def produce_from_json(self, data: json) -> dict: if not data: self.print_stderr('ERROR: No JSON data provided to parse.') return None - self.print_debug(f'Processing Scancode results into Dependency data...') + self.print_debug('Processing Scancode results into Dependency data...') files = [] for t in data: if t == 'files': # Only interested in 'files' details @@ -118,7 +118,6 @@ def produce_from_json(self, data: json) -> dict: continue self.print_debug(f'Path: {f_path}, Packages: {len(f_packages)}') purls = [] - scopes = [] for pkgs in f_packages: pk_deps = pkgs.get('dependencies') @@ -263,6 +262,7 @@ def run_scan(self, output_file: str = None, what_to_scan: str = None) -> bool: stderr=subprocess.STDOUT, text=True, timeout=self.timeout, + check=False, ) self.print_trace(f'Subprocess return: {result}') if result.returncode: From bb6be7f4236db09511b8cb8c2ed0b36bf9d536aa Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Mar 2026 16:29:36 +0100 Subject: [PATCH 461/489] refactor(cli)!: remove gRPC support --- CHANGELOG.md | 8 ++++- src/scanoss/cli.py | 73 +++++++++++++++++++++------------------------- 2 files changed, 40 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f03f7f1..3e1ba473 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [1.47.0] - 2026-03-05 ### Added -- Added support for skipping dependency files, configurable via `settings.skip.patterns.dependencies` +- Added support for skipping dependency files, configurable via `settings.skip.patterns.dependencies` + +### Changed +- All API communication now uses REST by default +- The `--grpc`, `--rest`, `--api2url`, and `--grpc-proxy` CLI flags now cause an error if used (gRPC is no longer supported) ## [1.46.0] - 2026-03-04 ### Added diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 8905c341..5440b810 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -1173,8 +1173,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--proxy', type=str, help='Proxy URL to use for connections (optional). ' - 'Can also use the environment variable "HTTPS_PROXY=:" ' - 'and "grcp_proxy=:" for gRPC', + 'Can also use the environment variable "HTTPS_PROXY=:"', ) p.add_argument( '--pac', @@ -1186,11 +1185,10 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 type=str, help='Alternative certificate PEM file (optional). ' 'Can also use the environment variable ' - '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem" and ' - '"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cacert.pem" for gRPC', + '"REQUESTS_CA_BUNDLE=/path/to/cacert.pem"', ) - # Global GRPC options + # Request options for p in [ p_scan, c_vulns, @@ -1206,13 +1204,12 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_licenses, ]: p.add_argument( - '--api2url', type=str, help='SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)' + '--api2url', type=str, + help='REMOVED: gRPC API URL is no longer supported. Using this flag will cause an error.' ) p.add_argument( - '--grpc-proxy', - type=str, - help='GRPC Proxy URL to use for connections (optional). ' - 'Can also use the environment variable "grcp_proxy=:"', + '--grpc-proxy', type=str, + help='REMOVED: gRPC Proxy URL is no longer supported. Using this flag will cause an error.' ) p.add_argument( '--header', @@ -1238,7 +1235,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Timeout (in seconds) for syft to complete (optional - default 600)', ) - # gRPC support options + # Deprecated gRPC/REST protocol flags (kept for clear error messaging) for p in [ c_vulns, p_scan, @@ -1253,8 +1250,14 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_licenses, p_folder_scan, ]: - p.add_argument('--grpc', action='store_true', default=True, help='Use gRPC (default)') - p.add_argument('--rest', action='store_true', dest='rest', help='Use REST instead of gRPC') + p.add_argument( + '--grpc', action='store_true', + help='REMOVED: gRPC is no longer supported. Using this flag will cause an error.' + ) + p.add_argument( + '--rest', action='store_true', dest='rest', + help='REMOVED: REST is now always used. Using this flag will cause an error.' + ) # Help/Trace command options for p in [ @@ -1304,10 +1307,24 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 args = parser.parse_args() - # TODO: Remove this hack once we go back to using REST as default - # Handle --rest overriding --grpc default + # Fail on removed gRPC-related flags + _removed_flags_used = [] + if hasattr(args, 'grpc') and args.grpc: + _removed_flags_used.append('--grpc') if hasattr(args, 'rest') and args.rest: - args.grpc = False + _removed_flags_used.append('--rest') + if hasattr(args, 'api2url') and args.api2url: + _removed_flags_used.append('--api2url') + if hasattr(args, 'grpc_proxy') and args.grpc_proxy: + _removed_flags_used.append('--grpc-proxy') + if _removed_flags_used: + flags_str = ', '.join(_removed_flags_used) + print_stderr( + f'Error: Removed flag(s) used: {flags_str}. ' + 'gRPC is no longer supported. REST is now used by default. ' + 'Please remove these flags from your command.' + ) + sys.exit(1) if args.version: ver(parser, args) @@ -1579,8 +1596,6 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 print_stderr('Obfuscating file fingerprints...') if args.proxy: print_stderr(f'Using Proxy {args.proxy}...') - if args.grpc_proxy: - print_stderr(f'Using GRPC Proxy {args.grpc_proxy}...') if args.pac: print_stderr(f'Using Proxy Auto-config (PAC) {args.pac}...') if args.ca_cert: @@ -1619,11 +1634,9 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command, - grpc_url=args.api2url, obfuscate=args.obfuscate, ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, - grpc_proxy=args.grpc_proxy, pac=pac_file, ca_cert=args.ca_cert, retry=args.retry, @@ -1636,7 +1649,6 @@ def scan(parser, args): # noqa: PLR0912, PLR0915 strip_snippet_ids=args.strip_snippet, scanoss_settings=scanoss_settings, req_headers=process_req_headers(args.header), - use_grpc=args.grpc, min_snippet_hits=args.min_snippet_hits, min_snippet_lines=args.min_snippet_lines, ranking=args.ranking, @@ -2431,16 +2443,13 @@ def comp_vulns(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - grpc_url=args.api2url, api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, - grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), ignore_cert_errors=args.ignore_cert_errors, - use_grpc=args.grpc, ) if not comps.get_vulnerabilities(args.input, args.purl, args.output): sys.exit(1) @@ -2468,15 +2477,12 @@ def comp_semgrep(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - grpc_url=args.api2url, api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, - grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), - use_grpc=args.grpc, ) if not comps.get_semgrep_details(args.input, args.purl, args.output): sys.exit(1) @@ -2507,15 +2513,12 @@ def comp_search(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - grpc_url=args.api2url, api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, - grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), - use_grpc=args.grpc, ) if not comps.search_components( args.output, @@ -2553,15 +2556,12 @@ def comp_versions(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - grpc_url=args.api2url, api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, - grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), - use_grpc=args.grpc, ) if not comps.get_component_versions(args.output, json_file=args.input, purl=args.purl, limit=args.limit): sys.exit(1) @@ -2589,15 +2589,12 @@ def comp_provenance(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - grpc_url=args.api2url, api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, - grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), - use_grpc=args.grpc, ) if not comps.get_provenance_details(args.input, args.purl, args.output, args.origin): sys.exit(1) @@ -2625,15 +2622,12 @@ def comp_licenses(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, - grpc_url=args.api2url, api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, - grpc_proxy=args.grpc_proxy, pac=pac_file, timeout=args.timeout, req_headers=process_req_headers(args.header), - use_grpc=args.grpc, ) if not comps.get_licenses(args.input, args.purl, args.output): sys.exit(1) @@ -2737,7 +2731,6 @@ def folder_hashing_scan(parser, args): depth=args.depth, recursive_threshold=args.recursive_threshold, min_accepted_score=args.min_accepted_score, - use_grpc=args.grpc, ) if scanner.scan(): From 9a8333ab665070e09c9fef62012cace5dd5c0475 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Mar 2026 16:38:38 +0100 Subject: [PATCH 462/489] docs(cli)!: remove references to gRPC support and related options --- CLIENT_HELP.md | 11 +++-------- docs/source/index.rst | 13 ------------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index ef4a36b8..23da2c77 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -52,12 +52,12 @@ The `scanoss-py` CLI uses two communication methods; REST & gRPC and as such req - `export GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/path/to/cert.pem` #### Custom Certificate with CLI Options -The `scanoss-py` CLI has a `--ca-cert` option to allow the specification of a custom certificate file to be used when communicating over REST/gRPC. +The `scanoss-py` CLI has a `--ca-cert` option to allow the specification of a custom certificate file to be used when communicating over REST. Simply set it using: ```shell scanoss-py scan --ca-cert scanoss-com.pem -o results.json . ``` -Alternative API Urls can also be configured (if necessary) using `--apiurl` & `api2url`. +Alternative API Urls can also be configured (if necessary) using `--apiurl`. #### Custom Certificate appended to Defaults It is also possible to append this custom certificate to the default certificate list used by `scanoss-py`. @@ -81,21 +81,16 @@ The SCANOSS clients can be configured to work with proxies. There are a number o There are a number of environment variables that can be specified to force the `scanoss-py` command to route calls via proxy. - REST - `https_proxy`, `http_proxy`, `HTTPS_PROXY`, `HTTP_PROXY` -- gRPC - `grpc_proxy`, `https_proxy`, `http_proxy` Set the variable as follows: `export https_proxy="http://:"` -The REST client support both lowercase & uppercase proxy names, however the gRPC client only supports lowercase variants. The gRPC client provides one extra variable, `grpc_proxy` to enable a separate proxy to be leveraged for it alone. +The REST client supports both lowercase & uppercase proxy names. ### Proxy CLI Options The proxy for REST based calls can also be configured directly on the `scanoss-py` commandline using `--proxy`. For example: ```shell scanoss-py scan --proxy "http://:" -o results.json . ``` -If a separate proxy is required for GRPC calls, please use the `--grpc-proxy` option: -```shell -scanoss-py scan --proxy "http://:" --grpc-proxy "http://:" -D -o results.json . -``` ### Proxy Auto-Config CLI Options The `scanoss-py` CLI also supports Proxy Auto-Config (PAC) when scanning using the `--pac` command option. diff --git a/docs/source/index.rst b/docs/source/index.rst index e3cacb16..c1515c4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -118,11 +118,6 @@ Scans a directory or file (source code or ``.wfp`` fingerprint file) and shows r - Proxy auto configuration (optional). * - --ca-cert - Alternative certificate PEM file, can also use the environment variables ``REQUEST_CA_BUNDLE`` and ``GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`` (optional) - * - --api2url - - SCANOSS gRPC API 2.0 base URL (optional - default https://api.osskb.org) - * - --grpc-proxy - - GRPC Proxy URL to use for connections, can also us the environment variable ``GRPC_PROXY`` (optional) - ------------------------------------------- Generate fingerprints: fingerprint, fp, wfp ------------------------------------------- @@ -264,10 +259,6 @@ Performs a comprehensive scan of a directory using folder hashing to identify co - Proxy auto configuration. Specify a file, http url or "auto" * - --ca-cert - Alternative certificate PEM file - * - --api2url - - SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org) - * - --grpc-proxy - - GRPC Proxy URL to use for connections -------------------------------- Folder Hashing: folder-hash, fh @@ -404,10 +395,6 @@ The following arguments are common to the ``algorithms``, ``hints``, and ``versi - Timeout (in seconds) for API communication (optional - default 600). * - --key , -k - SCANOSS API Key token (optional - not required for default OSSKB URL). - * - --api2url - - SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org). - * - --grpc-proxy - - GRPC Proxy URL to use for connections. * - --ca-cert - Alternative certificate PEM file. * - --debug, -d From f87d38665bba6870e1e6236825554e739485242b Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 5 Mar 2026 17:13:19 +0100 Subject: [PATCH 463/489] chore(release): bump version to 1.47.0 --- src/scanoss/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index cfb554be..7a66ad24 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.46.0' +__version__ = '1.47.0' From feff99acc8fb0a942e170c917b58265a57fa6d6e Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 6 Mar 2026 12:13:42 +0100 Subject: [PATCH 464/489] feat(cli): add --apiurl option to component subcommands The component subcommands (vulns, licenses, semgrep, provenance, search, versions) had no way to override the default API base URL. Customers with dedicated server endpoints could not use these commands against their assigned server, resulting in Unauthorized errors. Add --apiurl to all 6 component subcommands, consistent with the existing scan command, and pass it through to Components(grpc_url=). --- src/scanoss/cli.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 5440b810..03055ac2 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -502,6 +502,19 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 help='Timeout (in seconds) for API communication (optional - default 600)', ) + # Common Component sub-command API URL option + for p in [ + c_vulns, + c_search, + c_versions, + c_semgrep, + c_provenance, + c_licenses, + ]: + p.add_argument( + '--apiurl', type=str, help='SCANOSS API base URL (optional - default: https://api.osskb.org)' + ) + # Sub-command: utils p_util = subparsers.add_parser( 'utils', @@ -2443,6 +2456,7 @@ def comp_vulns(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, + grpc_url=args.apiurl, # Legacy param name; accepts the REST API base URL. TODO: rename to url api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, @@ -2477,6 +2491,7 @@ def comp_semgrep(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, + grpc_url=args.apiurl, # Legacy param name; accepts the REST API base URL. TODO: rename to url api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, @@ -2513,6 +2528,7 @@ def comp_search(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, + grpc_url=args.apiurl, # Legacy param name; accepts the REST API base URL. TODO: rename to url api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, @@ -2556,6 +2572,7 @@ def comp_versions(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, + grpc_url=args.apiurl, # Legacy param name; accepts the REST API base URL. TODO: rename to url api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, @@ -2589,6 +2606,7 @@ def comp_provenance(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, + grpc_url=args.apiurl, # Legacy param name; accepts the REST API base URL. TODO: rename to url api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, @@ -2622,6 +2640,7 @@ def comp_licenses(parser, args): debug=args.debug, trace=args.trace, quiet=args.quiet, + grpc_url=args.apiurl, # Legacy param name; accepts the REST API base URL. TODO: rename to url api_key=args.key, ca_cert=args.ca_cert, proxy=args.proxy, From 10edb1e432f1e31f301ffc74709f36731121b104 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 6 Mar 2026 12:15:32 +0100 Subject: [PATCH 465/489] docs(changelog): add --apiurl component subcommands entry and fix missing version links --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e1ba473..348b1719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Added `--apiurl` option to all component subcommands (`comp vulns`, `comp licenses`, `comp semgrep`, `comp provenance`, `comp search`, `comp versions`) to allow overriding the default API base URL ## [1.47.0] - 2026-03-05 ### Added @@ -823,4 +825,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.44.0]: https://github.com/scanoss/scanoss.py/compare/v1.43.1...v1.44.0 [1.45.0]: https://github.com/scanoss/scanoss.py/compare/v1.44.0...v1.45.0 [1.45.1]: https://github.com/scanoss/scanoss.py/compare/v1.45.0...v1.45.1 -[1.46.0]: https://github.com/scanoss/scanoss.py/compare/v1.45.1...v1.46.0 \ No newline at end of file +[1.46.0]: https://github.com/scanoss/scanoss.py/compare/v1.45.1...v1.46.0 +[1.47.0]: https://github.com/scanoss/scanoss.py/compare/v1.46.0...v1.47.0 From aea11950267f5fea75cb583792bbc28a3dd3f3e4 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 6 Mar 2026 12:19:44 +0100 Subject: [PATCH 466/489] chore(release): bump version to 1.48.0 --- CHANGELOG.md | 3 +++ src/scanoss/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 348b1719..517a2f81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [1.48.0] - 2026-03-06 ### Added - Added `--apiurl` option to all component subcommands (`comp vulns`, `comp licenses`, `comp semgrep`, `comp provenance`, `comp search`, `comp versions`) to allow overriding the default API base URL @@ -827,3 +829,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.45.1]: https://github.com/scanoss/scanoss.py/compare/v1.45.0...v1.45.1 [1.46.0]: https://github.com/scanoss/scanoss.py/compare/v1.45.1...v1.46.0 [1.47.0]: https://github.com/scanoss/scanoss.py/compare/v1.46.0...v1.47.0 +[1.48.0]: https://github.com/scanoss/scanoss.py/compare/v1.47.0...v1.48.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 7a66ad24..4287241c 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.47.0' +__version__ = '1.48.0' From 126bf94d4d4c3cb78067bd57bc1c0be905452829 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Wed, 11 Mar 2026 10:04:21 -0300 Subject: [PATCH 467/489] fix(skip-headers):SP-4119 fix skip header for multiple import lines --- scanoss.json | 4 +- src/scanoss/header_filter.py | 35 +- tests/data/.gitignore | 2 + tests/data/header-files-test/DataFrame.scala | 71 +++ tests/data/header-files-test/FileModel.ts | 152 +++++ tests/data/header-files-test/HttpClient.kt | 67 +++ tests/data/header-files-test/Package.swift | 70 +++ tests/data/header-files-test/Parser.hs | 103 ++++ tests/data/header-files-test/Router.php | 83 +++ .../data/header-files-test/ServiceProvider.cs | 109 ++++ tests/data/header-files-test/StringUtils.hpp | 89 +++ .../data/header-files-test/TokenVerifier.java | 538 +++++++++++++++++ tests/data/header-files-test/ViewController.m | 97 +++ tests/data/header-files-test/analysis.r | 79 +++ tests/data/header-files-test/cache.lua | 127 ++++ tests/data/header-files-test/config.rs | 103 ++++ tests/data/header-files-test/core.clj | 66 +++ tests/data/header-files-test/crc32c.c | 421 ++++++++++++++ tests/data/header-files-test/deploy.sh | 115 ++++ tests/data/header-files-test/handler.go | 84 +++ tests/data/header-files-test/logger.rb | 73 +++ .../header-files-test/multiline_imports.py | 99 ++++ tests/data/header-files-test/parser.pl | 94 +++ tests/data/header-files-test/results.py | 275 +++++++++ tests/data/header-files-test/server.ex | 100 ++++ tests/data/header-files-test/server.js | 90 +++ tests/data/header-files-test/widget.dart | 77 +++ tests/data/test_src_files.tar.gz | Bin 0 -> 29169 bytes tests/test_headers_filter.py | 550 ++++++++---------- 29 files changed, 3437 insertions(+), 336 deletions(-) create mode 100644 tests/data/.gitignore create mode 100644 tests/data/header-files-test/DataFrame.scala create mode 100644 tests/data/header-files-test/FileModel.ts create mode 100644 tests/data/header-files-test/HttpClient.kt create mode 100644 tests/data/header-files-test/Package.swift create mode 100644 tests/data/header-files-test/Parser.hs create mode 100644 tests/data/header-files-test/Router.php create mode 100644 tests/data/header-files-test/ServiceProvider.cs create mode 100644 tests/data/header-files-test/StringUtils.hpp create mode 100755 tests/data/header-files-test/TokenVerifier.java create mode 100644 tests/data/header-files-test/ViewController.m create mode 100644 tests/data/header-files-test/analysis.r create mode 100644 tests/data/header-files-test/cache.lua create mode 100644 tests/data/header-files-test/config.rs create mode 100644 tests/data/header-files-test/core.clj create mode 100644 tests/data/header-files-test/crc32c.c create mode 100644 tests/data/header-files-test/deploy.sh create mode 100644 tests/data/header-files-test/handler.go create mode 100644 tests/data/header-files-test/logger.rb create mode 100644 tests/data/header-files-test/multiline_imports.py create mode 100644 tests/data/header-files-test/parser.pl create mode 100644 tests/data/header-files-test/results.py create mode 100644 tests/data/header-files-test/server.ex create mode 100644 tests/data/header-files-test/server.js create mode 100644 tests/data/header-files-test/widget.dart create mode 100644 tests/data/test_src_files.tar.gz diff --git a/scanoss.json b/scanoss.json index 08f20a82..11f807bf 100644 --- a/scanoss.json +++ b/scanoss.json @@ -5,7 +5,9 @@ "scanning": [ "src/protoc_gen_swagger", "docs", - "scanoss_common_pb2_grpc.py" + "scanoss_common_pb2_grpc.py", + "tests/data/test_src_files.tar.gz", + "tests/data/src" ] }, "sizes": {} diff --git a/src/scanoss/header_filter.py b/src/scanoss/header_filter.py index d6fe10fe..0b6869dd 100644 --- a/src/scanoss/header_filter.py +++ b/src/scanoss/header_filter.py @@ -501,8 +501,8 @@ def find_first_implementation_line(self, lines: list[str], language: str) -> Opt return None in_multiline_comment = False in_license_section = False - in_import_block = False # To handle import blocks in Go - consecutive_imports_count = 0 + inside_multiline_import = False # To handle multi-line import blocks + found_imports = False # Get comment & import patterns for the language comment_patterns = self.patterns.COMMENT_PATTERNS[self.get_comment_style(language)] import_patterns = self.patterns.IMPORT_PATTERNS[language] @@ -532,26 +532,23 @@ def find_first_implementation_line(self, lines: list[str], language: str) -> Opt # If not a comment but we find a non-empty line, end license section if not is_a_comment: in_license_section = False - # Handle import blocks in Go - if language == 'go': - if stripped.startswith('import ('): - self.print_trace(f'Line {line_number}: Detected Go import block start') - in_import_block = True - continue - if in_import_block: - if stripped == ')': - self.print_trace(f'Line {line_number}: Detected Go import block end') - in_import_block = False - continue - if (stripped.startswith('"') or stripped.startswith('_') or - re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\s+"', stripped)): - # It's part of the import block - continue + # Check if this line closes a multi-line import block + if inside_multiline_import and (')' in stripped or '}' in stripped): + self.print_trace(f'Line {line_number}: Multi-line import block end') + inside_multiline_import = False + continue + # Skip continuation lines inside multi-line import blocks (e.g. "DEFAULT_SYFT_COMMAND,") + if inside_multiline_import: + continue # Check if it's an import if self.is_import(line, import_patterns): - if consecutive_imports_count == 0: + if not found_imports: self.print_trace(f'Line {line_number}: Detected import section') - consecutive_imports_count += 1 + found_imports = True + # Detect start of multi-line import block (e.g. "from x import (", "import {") + if ('(' in stripped and ')' not in stripped) or ('{' in stripped and '}' not in stripped): + self.print_trace(f'Line {line_number}: Multi-line import block start') + inside_multiline_import = True continue # If we get here, it's implementation code - return immediately! self.print_trace(f'Line {line_number}: First implementation line detected') diff --git a/tests/data/.gitignore b/tests/data/.gitignore new file mode 100644 index 00000000..fd14195c --- /dev/null +++ b/tests/data/.gitignore @@ -0,0 +1,2 @@ +src +!test_src_files.tar.gz \ No newline at end of file diff --git a/tests/data/header-files-test/DataFrame.scala b/tests/data/header-files-test/DataFrame.scala new file mode 100644 index 00000000..0b25eef6 --- /dev/null +++ b/tests/data/header-files-test/DataFrame.scala @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import scala.collection.mutable.ArrayBuffer +import scala.reflect.runtime.universe.TypeTag + +import org.apache.spark.annotation.{DeveloperApi, Evolving} +import org.apache.spark.api.java.function._ +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.types.StructType + +class DataFrame private[sql]( + @transient val sparkSession: SparkSession, + @DeveloperApi @Evolving val queryExecution: QueryExecution) + extends Dataset[Row] { + + def select(cols: Column*): DataFrame = { + val outputColumns = cols.map(_.named) + sparkSession.sessionState.executePlan( + Project(outputColumns, queryExecution.logical)) + .toDataFrame + } + + def filter(condition: Column): DataFrame = { + sparkSession.sessionState.executePlan( + Filter(condition.expr, queryExecution.logical)) + .toDataFrame + } + + def groupBy(cols: Column*): RelationalGroupedDataset = { + new RelationalGroupedDataset( + this, + cols.map(_.expr), + RelationalGroupedDataset.GroupByType) + } + + def join(right: DataFrame, joinExprs: Column): DataFrame = { + join(right, joinExprs, "inner") + } + + def join(right: DataFrame, joinExprs: Column, joinType: String): DataFrame = { + sparkSession.sessionState.executePlan( + Join(queryExecution.logical, right.queryExecution.logical, + JoinType(joinType), Some(joinExprs.expr), JoinHint.NONE)) + .toDataFrame + } + + def count(): Long = { + groupBy().count().collect().head.getLong(0) + } + + override def toString: String = { + s"DataFrame[${schema.map(f => s"${f.name}: ${f.dataType}").mkString(", ")}]" + } +} \ No newline at end of file diff --git a/tests/data/header-files-test/FileModel.ts b/tests/data/header-files-test/FileModel.ts new file mode 100644 index 00000000..9f2370b7 --- /dev/null +++ b/tests/data/header-files-test/FileModel.ts @@ -0,0 +1,152 @@ +import * as util from 'util'; +import sqlite3 from 'sqlite3'; +import { queries } from '../../querys_db'; +import { InventoryModel } from './InventoryModel'; +import { QueryBuilder } from '../../queryBuilder/QueryBuilder'; +import { Model } from '../../Model'; + +const { promisify } = require('util'); + +export class FileModel extends Model { + private connection: sqlite3.Database; + + public static readonly entityMapper = { + path: 'f.path', + purl: 'comp.purl', + version: 'comp.version', + source: 'comp.source', + id: 'fileId', + }; + + inventory: InventoryModel; + + constructor(conn: sqlite3.Database) { + super(); + this.connection = conn; + this.inventory = new InventoryModel(conn); + } + + public async get(queryBuilder: QueryBuilder) { + const SQLquery = this.getSQL( + queryBuilder, + 'SELECT f.fileId, f.path,(CASE WHEN f.identified=1 THEN \'IDENTIFIED\' WHEN f.identified=0 AND f.ignored=0 THEN \'PENDING\' ELSE \'ORIGINAL\' END) AS status, f.type FROM files f #FILTER;', + this.getEntityMapper(), + ); + const call = promisify(this.connection.get.bind(this.connection)); + const file = await call(SQLquery.SQL, ...SQLquery.params); + return file; + } + + public async getAll(queryBuilder?: QueryBuilder): Promise { + const SQLquery = this.getSQL(queryBuilder, queries.SQL_GET_ALL_FILES, this.getEntityMapper()); + const call = promisify(this.connection.all.bind(this.connection)); + const files = await call(SQLquery.SQL, SQLquery.params); + return files; + } + + public async getAllBySearch(queryBuilder?: QueryBuilder): Promise { + const SQLQuery = this.getSQL(queryBuilder, queries.SQL_GET_ALL_FILES_BY_SEARCH, this.getEntityMapper()); + const call:any = util.promisify(this.connection.all.bind(this.connection)); + const files = await call(SQLQuery.SQL, ...SQLQuery.params); + return files; + } + + public async ignored(files: number[]) { + const sql = `${queries.SQL_UPDATE_IGNORED_FILES}(${files.toString()});`; + const call = promisify(this.connection.run.bind(this.connection)); + await call(sql); + } + + public async insertFiles(data: Array) { + return new Promise(async (resolve, reject) => { + this.connection.serialize(async () => { + this.connection.run('begin transaction'); + for (let i = 0; i < data.length; i += 1) { + this.connection.run('INSERT INTO FILES(path,type) VALUES(?,?) ON CONFLICT(path) DO UPDATE SET type = excluded.type; ', data[i].path, data[i].type); + } + + this.connection.run('commit', (err: any) => { + if (!err) resolve(); + reject(err); + }); + }); + }); + } + + public async setDirty(dirty: number, path?: string) { + const call = promisify(this.connection.run.bind(this.connection)); + const SQLquery = path !== undefined ? `UPDATE files SET dirty=${dirty} WHERE path IN (${path});` : `UPDATE files SET dirty=${dirty};`; + await call(SQLquery); + } + + public async getDirty() { + const call = promisify(this.connection.all.bind(this.connection)); + const dirtyFiles = await call('SELECT fileId AS id FROM files WHERE dirty=1;'); + if (dirtyFiles) return dirtyFiles.map((item: any) => item.id); + return []; + } + + public async deleteDirty() { + const call = promisify(this.connection.run.bind(this.connection)); + await call('DELETE FROM files WHERE dirty=1;'); + } + + public async getClean() { + const call = promisify(this.connection.all.bind(this.connection)); + const files = await call('SELECT * FROM files WHERE dirty=0;'); + return files; + } + + public async getFilesRescan() { + const call = promisify(this.connection.all.bind(this.connection)); + const files = await call('SELECT f.path,f.identified ,f.ignored ,f.type AS original,(CASE WHEN f.identified=0 AND f.ignored=0 THEN 1 ELSE 0 END) as pending FROM files f;'); + return files; + } + + public async restore(files: number[]) { + const filesIds = `(${files.toString()});`; + const sql = queries.SQL_FILE_RESTORE + filesIds; + const call = promisify(this.connection.run.bind(this.connection)); + await call(sql); + } + + public async identified(ids: number[]) { + const call = promisify(this.connection.run.bind(this.connection)); + const resultsid = `(${ids.toString()});`; + const sql = queries.SQL_FILES_UPDATE_IDENTIFIED + resultsid; + await call(sql); + } + + public async updateFileType(fileIds: number[], fileType: string) { + const call = promisify(this.connection.run.bind(this.connection)); + const sql = `UPDATE files SET type=? WHERE fileId IN (${fileIds.toString()});`; + await call(sql, fileType); + } + + public async getSummary() { + const call = promisify(this.connection.get.bind(this.connection)); + const summary = await call(`SELECT COUNT(*) as totalFiles , (SELECT COUNT(*) FROM files WHERE type='MATCH') AS matchFiles, + (SELECT COUNT(*) FROM files WHERE type='FILTERED') AS filterFiles, + (SELECT COUNT(*) FROM files WHERE type='NO-MATCH') AS noMatchFiles, (SELECT COUNT(*) FROM files f WHERE f.type="MATCH" AND f.identified=1) AS scannedIdentified, + (SELECT COUNT(*) AS detectedIdentifiedFiles FROM files f WHERE f.identified=1) AS totalIdentified, + (SELECT COUNT(*) AS detectedIdentifiedFiles FROM files f WHERE f.ignored=1) AS original, + (SELECT COUNT(*) AS pending FROM files f WHERE f.identified=0 AND f.ignored=0 AND f.type="MATCH") AS pending FROM files;`); + return summary; + } + + public async getDetectedSummary() { + const call = promisify(this.connection.get.bind(this.connection)); + const summary = await call(`SELECT COUNT(*) as totalFiles , (SELECT COUNT(*) FROM files WHERE type='MATCH') AS matchFiles, + (SELECT COUNT(*) FROM files WHERE type='FILTERED') AS filterFiles, + (SELECT COUNT(*) FROM files WHERE type='NO-MATCH') AS noMatchFiles, + 0 AS scannedIdentified, + 0 AS totalIdentified, + 0 AS original, + (SELECT COUNT(*) AS pending FROM files f WHERE f.identified=0 AND f.ignored=0 AND f.type="MATCH") AS pending FROM files;`); + return summary; + } + + public getEntityMapper(): Record { + return FileModel.entityMapper; + } +} diff --git a/tests/data/header-files-test/HttpClient.kt b/tests/data/header-files-test/HttpClient.kt new file mode 100644 index 00000000..b2cb2015 --- /dev/null +++ b/tests/data/header-files-test/HttpClient.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ktor.client + +import io.ktor.client.engine.* +import io.ktor.client.plugins.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.util.* +import kotlinx.coroutines.* +import kotlin.coroutines.* + +class HttpClient( + private val engine: HttpClientEngine, + private val config: HttpClientConfig +) : CoroutineScope, Closeable { + + override val coroutineContext: CoroutineContext + get() = engine.coroutineContext + + private val plugins = mutableMapOf, Any>() + + suspend fun request(builder: HttpRequestBuilder): HttpResponse { + val call = engine.execute(builder) + return call.response + } + + suspend fun get(urlString: String, block: HttpRequestBuilder.() -> Unit = {}): HttpResponse { + return request { + url(urlString) + method = HttpMethod.Get + block() + } + } + + suspend fun post(urlString: String, block: HttpRequestBuilder.() -> Unit = {}): HttpResponse { + return request { + url(urlString) + method = HttpMethod.Post + block() + } + } + + fun plugin(key: AttributeKey): T { + @Suppress("UNCHECKED_CAST") + return plugins[key] as? T + ?: throw IllegalStateException("Plugin $key is not installed") + } + + override fun close() { + engine.close() + } +} \ No newline at end of file diff --git a/tests/data/header-files-test/Package.swift b/tests/data/header-files-test/Package.swift new file mode 100644 index 00000000..18dd0f35 --- /dev/null +++ b/tests/data/header-files-test/Package.swift @@ -0,0 +1,70 @@ +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import Foundation +import Combine +import os.log + +/// A thread-safe container that manages the lifecycle of registered services. +/// +/// `ServiceContainer` provides dependency injection capabilities by storing +/// factory closures that create service instances on demand. Services can be +/// registered as transient (new instance each time) or singleton (shared instance). +public final class ServiceContainer: @unchecked Sendable { + private let lock = NSRecursiveLock() + private var factories: [String: () -> Any] = [:] + private var singletons: [String: Any] = [:] + private let logger = Logger(subsystem: "com.app", category: "DI") + + public static let shared = ServiceContainer() + + public init() {} + + /// Registers a factory closure for the given type. + public func register(_ type: T.Type, factory: @escaping () -> T) { + let key = String(describing: type) + lock.lock() + defer { lock.unlock() } + factories[key] = factory + logger.debug("Registered service: \(key)") + } + + /// Registers a singleton instance for the given type. + public func registerSingleton(_ type: T.Type, factory: @escaping () -> T) { + let key = String(describing: type) + lock.lock() + defer { lock.unlock() } + singletons[key] = factory() + logger.debug("Registered singleton: \(key)") + } + + /// Resolves an instance of the given type. + public func resolve(_ type: T.Type) -> T? { + let key = String(describing: type) + lock.lock() + defer { lock.unlock() } + + if let singleton = singletons[key] as? T { + return singleton + } + + guard let factory = factories[key] else { + logger.warning("No registration found for: \(key)") + return nil + } + + return factory() as? T + } + + /// Removes all registrations. + public func reset() { + lock.lock() + defer { lock.unlock() } + factories.removeAll() + singletons.removeAll() + logger.info("Container reset") + } +} \ No newline at end of file diff --git a/tests/data/header-files-test/Parser.hs b/tests/data/header-files-test/Parser.hs new file mode 100644 index 00000000..d25847d9 --- /dev/null +++ b/tests/data/header-files-test/Parser.hs @@ -0,0 +1,103 @@ +-- | +-- Module : Text.Parsec.Combinator +-- Copyright : (c) 2024 Daan Leijen, Paolo Martini +-- License : BSD-3-Clause +-- +-- Maintainer : asr@eafit.edu.co +-- Stability : provisional +-- Portability : portable +-- +-- Commonly used generic parser combinators. + +module Text.Parsec.Combinator + ( choice + , count + , between + , option + , optional + , sepBy + , sepBy1 + , many1 + , chainl + , chainl1 + , chainr + , chainr1 + , eof + , notFollowedBy + , manyTill + , lookAhead + ) where + +import Text.Parsec.Prim (Parser, (<|>), try, unexpected, lookAhead) + +-- | @choice ps@ tries to apply the parsers in the list @ps@ in order, +-- until one of them succeeds. Returns the value of the succeeding parser. +choice :: [Parser a] -> Parser a +choice = foldr (<|>) (unexpected "no match") + +-- | @count n p@ parses @n@ occurrences of @p@. +count :: Int -> Parser a -> Parser [a] +count n p + | n <= 0 = return [] + | otherwise = sequence (replicate n p) + +-- | @between open close p@ parses @open@, followed by @p@ and @close@. +-- Returns the value returned by @p@. +between :: Parser open -> Parser close -> Parser a -> Parser a +between open close p = do + _ <- open + x <- p + _ <- close + return x + +-- | @option x p@ tries to apply parser @p@. If @p@ fails without +-- consuming input, it returns the value @x@, otherwise the value +-- returned by @p@. +option :: a -> Parser a -> Parser a +option x p = p <|> return x + +-- | @optional p@ tries to apply parser @p@. It will parse @p@ or nothing. +-- It only fails if @p@ fails after consuming input. +optional :: Parser a -> Parser () +optional p = (p >> return ()) <|> return () + +-- | @sepBy p sep@ parses zero or more occurrences of @p@, separated +-- by @sep@. Returns a list of values returned by @p@. +sepBy :: Parser a -> Parser sep -> Parser [a] +sepBy p sep = sepBy1 p sep <|> return [] + +-- | @sepBy1 p sep@ parses one or more occurrences of @p@, separated +-- by @sep@. Returns a list of values returned by @p@. +sepBy1 :: Parser a -> Parser sep -> Parser [a] +sepBy1 p sep = do + x <- p + xs <- many (sep >> p) + return (x : xs) + +-- | @many1 p@ applies the parser @p@ one or more times. +many1 :: Parser a -> Parser [a] +many1 p = do + x <- p + xs <- many p + return (x : xs) + +-- | @chainl p op x@ parses zero or more occurrences of @p@, +-- separated by @op@. Returns a value obtained by a left associative +-- application of all functions returned by @op@ to the values +-- returned by @p@. If there are zero occurrences of @p@, the value +-- @x@ is returned. +chainl :: Parser a -> Parser (a -> a -> a) -> a -> Parser a +chainl p op x = chainl1 p op <|> return x + +-- | @chainl1 p op@ parses one or more occurrences of @p@, +-- separated by @op@. Returns a value obtained by a left associative +-- application of all functions returned by @op@. +chainl1 :: Parser a -> Parser (a -> a -> a) -> Parser a +chainl1 p op = do + x <- p + rest x + where + rest x = (do f <- op + y <- p + rest (f x y)) + <|> return x \ No newline at end of file diff --git a/tests/data/header-files-test/Router.php b/tests/data/header-files-test/Router.php new file mode 100644 index 00000000..9fb92a60 --- /dev/null +++ b/tests/data/header-files-test/Router.php @@ -0,0 +1,83 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Routing; + +use Symfony\Component\HttpFoundation\Request; +use Symfony\Component\Routing\Exception\MethodNotAllowedException; +use Symfony\Component\Routing\Exception\ResourceNotFoundException; +use Symfony\Component\Routing\Matcher\UrlMatcherInterface; + +class Router implements RouterInterface +{ + private RouteCollection $routes; + private UrlMatcherInterface $matcher; + private array $options; + + public function __construct(RouteCollection $routes, array $options = []) + { + $this->routes = $routes; + $this->options = array_merge([ + 'cache_dir' => null, + 'debug' => false, + 'strict_requirements' => true, + ], $options); + } + + public function match(string $pathinfo): array + { + return $this->getMatcher()->match($pathinfo); + } + + public function matchRequest(Request $request): array + { + $pathinfo = $request->getPathInfo(); + $method = $request->getMethod(); + + try { + $parameters = $this->match($pathinfo); + } catch (ResourceNotFoundException $e) { + throw new ResourceNotFoundException( + sprintf('No route found for "%s %s"', $method, $pathinfo), + 0, + $e + ); + } + + if (isset($parameters['_method'])) { + $allowedMethods = explode('|', $parameters['_method']); + if (!in_array($method, $allowedMethods, true)) { + throw new MethodNotAllowedException($allowedMethods); + } + } + + return $parameters; + } + + public function getRouteCollection(): RouteCollection + { + return $this->routes; + } + + public function addRoute(string $name, Route $route): void + { + $this->routes->add($name, $route); + } + + private function getMatcher(): UrlMatcherInterface + { + if (!isset($this->matcher)) { + $this->matcher = new UrlMatcher($this->routes, new RequestContext()); + } + + return $this->matcher; + } +} \ No newline at end of file diff --git a/tests/data/header-files-test/ServiceProvider.cs b/tests/data/header-files-test/ServiceProvider.cs new file mode 100644 index 00000000..f68f38c3 --- /dev/null +++ b/tests/data/header-files-test/ServiceProvider.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; + +namespace Microsoft.Extensions.DependencyInjection +{ + /// + /// The default IServiceProvider implementation. + /// + public sealed class ServiceProvider : IServiceProvider, IDisposable, IAsyncDisposable + { + private readonly ConcurrentDictionary> _factories; + private readonly ConcurrentDictionary _singletons; + private readonly IServiceCollection _services; + private bool _disposed; + + internal ServiceProvider(IServiceCollection services) + { + _services = services ?? throw new ArgumentNullException(nameof(services)); + _factories = new ConcurrentDictionary>(); + _singletons = new ConcurrentDictionary(); + } + + /// + /// Gets the service object of the specified type. + /// + public object GetService(Type serviceType) + { + if (_disposed) + throw new ObjectDisposedException(nameof(ServiceProvider)); + + if (serviceType == typeof(IServiceProvider)) + return this; + + if (_singletons.TryGetValue(serviceType, out var singleton)) + return singleton; + + if (_factories.TryGetValue(serviceType, out var factory)) + return factory(this); + + var descriptor = _services.FirstOrDefault(d => d.ServiceType == serviceType); + if (descriptor == null) + return null; + + return CreateInstance(descriptor); + } + + private object CreateInstance(ServiceDescriptor descriptor) + { + if (descriptor.ImplementationInstance != null) + return descriptor.ImplementationInstance; + + if (descriptor.ImplementationFactory != null) + return descriptor.ImplementationFactory(this); + + var implementationType = descriptor.ImplementationType; + var constructor = implementationType.GetConstructors().OrderByDescending(c => c.GetParameters().Length).First(); + + var parameters = constructor.GetParameters() + .Select(p => GetService(p.ParameterType)) + .ToArray(); + + return constructor.Invoke(parameters); + } + + public void Dispose() + { + if (_disposed) + return; + + _disposed = true; + + foreach (var singleton in _singletons.Values) + { + if (singleton is IDisposable disposable) + disposable.Dispose(); + } + + _singletons.Clear(); + _factories.Clear(); + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + return; + + _disposed = true; + + foreach (var singleton in _singletons.Values) + { + if (singleton is IAsyncDisposable asyncDisposable) + await asyncDisposable.DisposeAsync(); + else if (singleton is IDisposable disposable) + disposable.Dispose(); + } + + _singletons.Clear(); + _factories.Clear(); + } + } +} \ No newline at end of file diff --git a/tests/data/header-files-test/StringUtils.hpp b/tests/data/header-files-test/StringUtils.hpp new file mode 100644 index 00000000..9a401900 --- /dev/null +++ b/tests/data/header-files-test/StringUtils.hpp @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2024 LLVM Project Contributors +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. + +#ifndef LLVM_ADT_STRING_UTILS_HPP +#define LLVM_ADT_STRING_UTILS_HPP + +#include +#include +#include +#include +#include +#include + +namespace llvm { + +/// Splits a string by the given delimiter and returns a vector of substrings. +/// +/// \param input The string to split. +/// \param delimiter The character to split on. +/// \return A vector of substrings. +inline std::vector split(const std::string &input, char delimiter) { + std::vector tokens; + std::istringstream stream(input); + std::string token; + + while (std::getline(stream, token, delimiter)) { + if (!token.empty()) { + tokens.push_back(token); + } + } + return tokens; +} + +/// Trims whitespace from the beginning and end of a string. +inline std::string trim(const std::string &str) { + auto start = std::find_if_not(str.begin(), str.end(), ::isspace); + auto end = std::find_if_not(str.rbegin(), str.rend(), ::isspace).base(); + + if (start >= end) { + return ""; + } + return std::string(start, end); +} + +/// Converts a string to lowercase. +inline std::string toLower(const std::string &str) { + std::string result = str; + std::transform(result.begin(), result.end(), result.begin(), ::tolower); + return result; +} + +/// Converts a string to uppercase. +inline std::string toUpper(const std::string &str) { + std::string result = str; + std::transform(result.begin(), result.end(), result.begin(), ::toupper); + return result; +} + +/// Checks if a string starts with the given prefix. +inline bool startsWith(const std::string &str, const std::string &prefix) { + return str.size() >= prefix.size() && + str.compare(0, prefix.size(), prefix) == 0; +} + +/// Checks if a string ends with the given suffix. +inline bool endsWith(const std::string &str, const std::string &suffix) { + return str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +/// Replaces all occurrences of a substring with another substring. +inline std::string replaceAll(const std::string &str, + const std::string &from, + const std::string &to) { + std::string result = str; + size_t pos = 0; + while ((pos = result.find(from, pos)) != std::string::npos) { + result.replace(pos, from.length(), to); + pos += to.length(); + } + return result; +} + +} // namespace llvm + +#endif // LLVM_ADT_STRING_UTILS_HPP \ No newline at end of file diff --git a/tests/data/header-files-test/TokenVerifier.java b/tests/data/header-files-test/TokenVerifier.java new file mode 100755 index 00000000..90b6e3e1 --- /dev/null +++ b/tests/data/header-files-test/TokenVerifier.java @@ -0,0 +1,538 @@ +/* + * Copyright 2016 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak; + +import java.nio.charset.StandardCharsets; +import java.security.PublicKey; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.crypto.SecretKey; + +import org.keycloak.common.VerificationException; +import org.keycloak.crypto.SignatureVerifierContext; +import org.keycloak.exceptions.TokenNotActiveException; +import org.keycloak.exceptions.TokenSignatureInvalidException; +import org.keycloak.jose.jws.AlgorithmType; +import org.keycloak.jose.jws.JWSHeader; +import org.keycloak.jose.jws.JWSInput; +import org.keycloak.jose.jws.JWSInputException; +import org.keycloak.jose.jws.crypto.HMACProvider; +import org.keycloak.jose.jws.crypto.RSAProvider; +import org.keycloak.representations.JsonWebToken; +import org.keycloak.util.TokenUtil; + +/** + * @author
Bill Burke + * @version $Revision: 1 $ + */ +public class TokenVerifier { + + private static final Logger LOG = Logger.getLogger(TokenVerifier.class.getName()); + + // This interface is here as JDK 7 is a requirement for this project. + // Once JDK 8 would become mandatory, java.util.function.Predicate would be used instead. + + /** + * Functional interface of checks that verify some part of a JWT. + * @param Type of the token handled by this predicate. + */ + // @FunctionalInterface + public static interface Predicate { + /** + * Performs a single check on the given token verifier. + * @param t Token, guaranteed to be non-null. + * @return + * @throws VerificationException + */ + boolean test(T t) throws VerificationException; + } + + public static final Predicate SUBJECT_EXISTS_CHECK = new Predicate() { + @Override + public boolean test(JsonWebToken t) throws VerificationException { + String subject = t.getSubject(); + if (subject == null) { + throw new VerificationException("Subject missing in token"); + } + + return true; + } + }; + + /** + * Check for token being neither expired nor used before it gets valid. + * @see JsonWebToken#isActive() + */ + public static final Predicate IS_ACTIVE = new Predicate() { + @Override + public boolean test(JsonWebToken t) throws VerificationException { + if (! t.isActive()) { + throw new TokenNotActiveException(t, "Token is not active"); + } + + return true; + } + }; + + public static class RealmUrlCheck implements Predicate { + + private static final RealmUrlCheck NULL_INSTANCE = new RealmUrlCheck(null); + + private final String realmUrl; + + public RealmUrlCheck(String realmUrl) { + this.realmUrl = realmUrl; + } + + @Override + public boolean test(JsonWebToken t) throws VerificationException { + if (this.realmUrl == null) { + throw new VerificationException("Realm URL not set"); + } + + if (! this.realmUrl.equals(t.getIssuer())) { + throw new VerificationException("Invalid token issuer. Expected '" + this.realmUrl + "'"); + } + + return true; + } + } + + public static class TokenTypeCheck implements Predicate { + + private static final TokenTypeCheck INSTANCE_DEFAULT_TOKEN_TYPE = new TokenTypeCheck(Arrays.asList(TokenUtil.TOKEN_TYPE_BEARER)); + + private final List tokenTypes; + + public TokenTypeCheck(List tokenTypes) { + this.tokenTypes = tokenTypes; + } + + @Override + public boolean test(JsonWebToken t) throws VerificationException { + for (String tokenType : tokenTypes) { + if (tokenType.equalsIgnoreCase(t.getType())) return true; + } + throw new VerificationException("Token type is incorrect. Expected '" + tokenTypes.toString() + "' but was '" + t.getType() + "'"); + } + } + + + public static class AudienceCheck implements Predicate { + + private final String expectedAudience; + + public AudienceCheck(String expectedAudience) { + this.expectedAudience = expectedAudience; + } + + @Override + public boolean test(JsonWebToken t) throws VerificationException { + if (expectedAudience == null) { + throw new VerificationException("Missing expectedAudience"); + } + + String[] audience = t.getAudience(); + if (audience == null) { + throw new VerificationException("No audience in the token"); + } + + if (t.hasAudience(expectedAudience)) { + return true; + } + + throw new VerificationException("Expected audience not available in the token"); + } + } + + + public static class IssuedForCheck implements Predicate { + + private final String expectedIssuedFor; + + public IssuedForCheck(String expectedIssuedFor) { + this.expectedIssuedFor = expectedIssuedFor; + } + + @Override + public boolean test(JsonWebToken jsonWebToken) throws VerificationException { + if (expectedIssuedFor == null) { + throw new VerificationException("Missing expectedIssuedFor"); + } + + if (expectedIssuedFor.equals(jsonWebToken.getIssuedFor())) { + return true; + } + + throw new VerificationException("Expected issuedFor doesn't match"); + } + } + + + private String tokenString; + private Class clazz; + private PublicKey publicKey; + private SecretKey secretKey; + private String realmUrl; + private List expectedTokenType = Arrays.asList(TokenUtil.TOKEN_TYPE_BEARER, TokenUtil.TOKEN_TYPE_DPOP); + private boolean checkTokenType = true; + private boolean checkRealmUrl = true; + private final LinkedList> checks = new LinkedList<>(); + + private JWSInput jws; + private T token; + + private SignatureVerifierContext verifier = null; + + public TokenVerifier verifierContext(SignatureVerifierContext verifier) { + this.verifier = verifier; + return this; + } + + protected TokenVerifier(String tokenString, Class clazz) { + this.tokenString = tokenString; + this.clazz = clazz; + } + + protected TokenVerifier(T token) { + this.token = token; + } + + /** + * Creates an instance of {@code TokenVerifier} from the given string on a JWT of the given class. + * The token verifier has no checks defined. Note that the checks are only tested when + * {@link #verify()} method is invoked. + * @param Type of the token + * @param tokenString String representation of JWT + * @param clazz Class of the token + * @return + */ + public static TokenVerifier create(String tokenString, Class clazz) { + return new TokenVerifier<>(tokenString, clazz); + } + + /** + * Creates an instance of {@code TokenVerifier} for the given token. + * The token verifier has no checks defined. Note that the checks are only tested when + * {@link #verify()} method is invoked. + *

+ * NOTE: The returned token verifier cannot verify token signature since + * that is not part of the {@link JsonWebToken} object. + * @return + */ + public static TokenVerifier createWithoutSignature(T token) { + return new TokenVerifier<>(token); + } + + /** + * Adds default checks to the token verification: + *

    + *
  • Realm URL (JWT issuer field: {@code iss}) has to be defined and match realm set via {@link #realmUrl(java.lang.String)} method
  • + *
  • Subject (JWT subject field: {@code sub}) has to be defined
  • + *
  • Token type (JWT type field: {@code typ}) has to be {@code Bearer}. The type can be set via {@link #tokenType(List)} method
  • + *
  • Token has to be active, ie. both not expired and not used before its validity (JWT issuer fields: {@code exp} and {@code nbf})
  • + *
+ * @return This token verifier. + */ + public TokenVerifier withDefaultChecks() { + return withChecks( + RealmUrlCheck.NULL_INSTANCE, + TokenTypeCheck.INSTANCE_DEFAULT_TOKEN_TYPE, + IS_ACTIVE + ); + } + + private void removeCheck(Class> checkClass) { + for (Iterator> it = checks.iterator(); it.hasNext();) { + if (it.next().getClass() == checkClass) { + it.remove(); + } + } + } + + private void removeCheck(Predicate check) { + checks.remove(check); + } + + @SuppressWarnings("unchecked") + private

> TokenVerifier replaceCheck(Class> checkClass, boolean active, P... predicate) { + removeCheck(checkClass); + if (active) { + checks.addAll(Arrays.asList(predicate)); + } + return this; + } + + @SuppressWarnings("unchecked") + private

> TokenVerifier replaceCheck(Predicate check, boolean active, P... predicate) { + removeCheck(check); + if (active) { + checks.addAll(Arrays.asList(predicate)); + } + return this; + } + + /** + * Will test the given checks in {@link #verify()} method in addition to already set checks. + * @param checks + * @return + */ + @SafeVarargs + public final TokenVerifier withChecks(Predicate... checks) { + if (checks != null) { + this.checks.addAll(Arrays.asList(checks)); + } + return this; + } + + /** + * Sets the key for verification of RSA-based signature. + * @param publicKey + * @return + */ + public TokenVerifier publicKey(PublicKey publicKey) { + this.publicKey = publicKey; + return this; + } + + /** + * Sets the key for verification of HMAC-based signature. + * @param secretKey + * @return + */ + public TokenVerifier secretKey(SecretKey secretKey) { + this.secretKey = secretKey; + return this; + } + + /** + * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. + * @return This token verifier + */ + public TokenVerifier realmUrl(String realmUrl) { + this.realmUrl = realmUrl; + return replaceCheck(RealmUrlCheck.class, checkRealmUrl, new RealmUrlCheck(realmUrl)); + } + + /** + * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. + * @return This token verifier + */ + public TokenVerifier checkTokenType(boolean checkTokenType) { + this.checkTokenType = checkTokenType; + return replaceCheck(TokenTypeCheck.class, this.checkTokenType, new TokenTypeCheck(expectedTokenType)); + } + + /** + * + * @return This token verifier + */ + public TokenVerifier tokenType(List tokenTypes) { + this.expectedTokenType = tokenTypes; + return replaceCheck(TokenTypeCheck.class, this.checkTokenType, new TokenTypeCheck(expectedTokenType)); + } + + /** + * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. + * @return This token verifier + */ + public TokenVerifier checkActive(boolean checkActive) { + return replaceCheck(IS_ACTIVE, checkActive, IS_ACTIVE); + } + + /** + * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. + * @return This token verifier + */ + public TokenVerifier checkRealmUrl(boolean checkRealmUrl) { + this.checkRealmUrl = checkRealmUrl; + return replaceCheck(RealmUrlCheck.class, this.checkRealmUrl, new RealmUrlCheck(realmUrl)); + } + + /** + * Add check for verifying that token contains the expectedAudience + * + * @param expectedAudiences Audiences, which needs to be in the target token. Can be null. + * @return This token verifier + */ + public TokenVerifier audience(String... expectedAudiences) { + if (expectedAudiences == null || expectedAudiences.length == 0) { + return this.replaceCheck(AudienceCheck.class, true, new AudienceCheck(null)); + } + AudienceCheck[] audienceChecks = new AudienceCheck[expectedAudiences.length]; + for (int i = 0; i < expectedAudiences.length; ++i) { + audienceChecks[i] = new AudienceCheck(expectedAudiences[i]); + } + return this.replaceCheck(AudienceCheck.class, true, audienceChecks); + } + + /** + * Add check for verifying that token issuedFor (azp claim) is the expected value + * + * @param expectedIssuedFor issuedFor, which needs to be in the target token. Can't be null + * @return This token verifier + */ + public TokenVerifier issuedFor(String expectedIssuedFor) { + return this.replaceCheck(IssuedForCheck.class, true, new IssuedForCheck(expectedIssuedFor)); + } + + public TokenVerifier parse() throws VerificationException { + if (jws == null) { + if (tokenString == null) { + throw new VerificationException("Token not set"); + } + + try { + jws = new JWSInput(tokenString); + } catch (JWSInputException e) { + throw new VerificationException("Failed to parse JWT", e); + } + + + try { + token = jws.readJsonContent(clazz); + } catch (JWSInputException e) { + throw new VerificationException("Failed to read access token from JWT", e); + } + } + return this; + } + + public T getToken() throws VerificationException { + if (token == null) { + parse(); + } + return token; + } + + public JWSHeader getHeader() throws VerificationException { + parse(); + return jws.getHeader(); + } + + public void verifySignature() throws VerificationException { + if (this.verifier != null) { + try { + if (!verifier.verify(jws.getEncodedSignatureInput().getBytes(StandardCharsets.UTF_8), jws.getSignature())) { + throw new TokenSignatureInvalidException(token, "Invalid token signature"); + } + } catch (Exception e) { + throw new VerificationException(e); + } + } else { + AlgorithmType algorithmType = getHeader().getAlgorithm().getType(); + + if (null == algorithmType) { + throw new VerificationException("Unknown or unsupported token algorithm"); + } else switch (algorithmType) { + case RSA: + if (publicKey == null) { + throw new VerificationException("Public key not set"); + } + if (!RSAProvider.verify(jws, publicKey)) { + throw new TokenSignatureInvalidException(token, "Invalid token signature"); + } + break; + case HMAC: + if (secretKey == null) { + throw new VerificationException("Secret key not set"); + } + if (!HMACProvider.verify(jws, secretKey)) { + throw new TokenSignatureInvalidException(token, "Invalid token signature"); + } + break; + default: + throw new VerificationException("Unknown or unsupported token algorithm"); + } + } + } + + public TokenVerifier verify() throws VerificationException { + if (getToken() == null) { + parse(); + } + if (jws != null) { + verifySignature(); + } + + for (Predicate check : checks) { + if (! check.test(getToken())) { + throw new VerificationException("JWT check failed for check " + check); + } + } + + return this; + } + + /** + * Creates an optional predicate from a predicate that will proceed with check but always pass. + * @param + * @param mandatoryPredicate + * @return + */ + public static Predicate optional(final Predicate mandatoryPredicate) { + return new Predicate() { + @Override + public boolean test(T t) throws VerificationException { + try { + if (! mandatoryPredicate.test(t)) { + LOG.finer("[optional] predicate failed: " + mandatoryPredicate); + } + + return true; + } catch (VerificationException ex) { + LOG.log(Level.FINER, "[optional] predicate " + mandatoryPredicate + " failed.", ex); + return true; + } + } + }; + } + + /** + * Creates a predicate that will proceed with checks of the given predicates + * and will pass if and only if at least one of the given predicates passes. + * @param + * @param predicates + * @return + */ + @SafeVarargs + public static Predicate alternative(final Predicate... predicates) { + return new Predicate() { + @Override + public boolean test(T t) { + for (Predicate predicate : predicates) { + try { + if (predicate.test(t)) { + return true; + } + + LOG.finer("[alternative] predicate failed: " + predicate); + } catch (VerificationException ex) { + LOG.log(Level.FINER, "[alternative] predicate " + predicate + " failed.", ex); + } + } + + return false; + } + }; + } +} diff --git a/tests/data/header-files-test/ViewController.m b/tests/data/header-files-test/ViewController.m new file mode 100644 index 00000000..16b9090c --- /dev/null +++ b/tests/data/header-files-test/ViewController.m @@ -0,0 +1,97 @@ +// Copyright (c) 2024 Apple Inc. +// Licensed under the Apache License, Version 2.0. +// See LICENSE file in the project root for full license information. +// +// SPDX-License-Identifier: Apache-2.0 + +#import "ViewController.h" +#import +#import + +@interface ViewController () + +@property (nonatomic, strong) UITableView *tableView; +@property (nonatomic, strong) NSMutableArray *dataSource; +@property (nonatomic, strong) UIRefreshControl *refreshControl; + +@end + +@implementation ViewController + +- (void)viewDidLoad { + [super viewDidLoad]; + self.title = @"Items"; + self.dataSource = [NSMutableArray array]; + + [self setupTableView]; + [self setupRefreshControl]; + [self loadData]; +} + +- (void)setupTableView { + self.tableView = [[UITableView alloc] initWithFrame:self.view.bounds + style:UITableViewStylePlain]; + self.tableView.dataSource = self; + self.tableView.delegate = self; + self.tableView.autoresizingMask = UIViewAutoresizingFlexibleWidth | + UIViewAutoresizingFlexibleHeight; + [self.tableView registerClass:[UITableViewCell class] + forCellReuseIdentifier:@"Cell"]; + [self.view addSubview:self.tableView]; +} + +- (void)setupRefreshControl { + self.refreshControl = [[UIRefreshControl alloc] init]; + [self.refreshControl addTarget:self + action:@selector(refreshData) + forControlEvents:UIControlEventValueChanged]; + self.tableView.refreshControl = self.refreshControl; +} + +- (void)loadData { + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ + NSArray *items = @[ + @{@"title": @"First Item", @"subtitle": @"Description 1"}, + @{@"title": @"Second Item", @"subtitle": @"Description 2"}, + @{@"title": @"Third Item", @"subtitle": @"Description 3"}, + ]; + + dispatch_async(dispatch_get_main_queue(), ^{ + [self.dataSource removeAllObjects]; + [self.dataSource addObjectsFromArray:items]; + [self.tableView reloadData]; + [self.refreshControl endRefreshing]; + }); + }); +} + +- (void)refreshData { + [self loadData]; +} + +#pragma mark - UITableViewDataSource + +- (NSInteger)tableView:(UITableView *)tableView numberOfRowsInSection:(NSInteger)section { + return self.dataSource.count; +} + +- (UITableViewCell *)tableView:(UITableView *)tableView + cellForRowAtIndexPath:(NSIndexPath *)indexPath { + UITableViewCell *cell = [tableView dequeueReusableCellWithIdentifier:@"Cell" + forIndexPath:indexPath]; + NSDictionary *item = self.dataSource[indexPath.row]; + cell.textLabel.text = item[@"title"]; + cell.detailTextLabel.text = item[@"subtitle"]; + cell.accessoryType = UITableViewCellAccessoryDisclosureIndicator; + return cell; +} + +#pragma mark - UITableViewDelegate + +- (void)tableView:(UITableView *)tableView didSelectRowAtIndexPath:(NSIndexPath *)indexPath { + [tableView deselectRowAtIndexPath:indexPath animated:YES]; + NSDictionary *item = self.dataSource[indexPath.row]; + NSLog(@"Selected: %@", item[@"title"]); +} + +@end \ No newline at end of file diff --git a/tests/data/header-files-test/analysis.r b/tests/data/header-files-test/analysis.r new file mode 100644 index 00000000..c2a7bc71 --- /dev/null +++ b/tests/data/header-files-test/analysis.r @@ -0,0 +1,79 @@ +# Copyright (c) 2024 The R Foundation for Statistical Computing +# +# This file is part of R, which is free software. You can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# R is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +library(ggplot2) +library(dplyr) +library(tidyr) +require(stats) + +#' Perform statistical analysis on the given dataset +#' +#' @param data A data frame containing the input data +#' @param target_col The name of the target variable column +#' @param predictor_cols A character vector of predictor column names +#' @return A list containing model summary and diagnostics +analyze_regression <- function(data, target_col, predictor_cols) { + if (!is.data.frame(data)) { + stop("Input must be a data frame") + } + + formula_str <- paste(target_col, "~", paste(predictor_cols, collapse = " + ")) + model <- lm(as.formula(formula_str), data = data) + + residuals <- residuals(model) + fitted_vals <- fitted(model) + + diagnostics <- list( + shapiro_test = shapiro.test(residuals), + r_squared = summary(model)$r.squared, + adj_r_squared = summary(model)$adj.r.squared, + f_statistic = summary(model)$fstatistic + ) + + result <- list( + model = model, + summary = summary(model), + diagnostics = diagnostics + ) + + return(result) +} + +plot_diagnostics <- function(model, output_dir = "plots") { + if (!dir.exists(output_dir)) { + dir.create(output_dir, recursive = TRUE) + } + + residual_data <- data.frame( + fitted = fitted(model), + residuals = residuals(model), + standardized = rstandard(model) + ) + + p1 <- ggplot(residual_data, aes(x = fitted, y = residuals)) + + geom_point(alpha = 0.5) + + geom_hline(yintercept = 0, linetype = "dashed", color = "red") + + labs(title = "Residuals vs Fitted", x = "Fitted Values", y = "Residuals") + + theme_minimal() + + ggsave(file.path(output_dir, "residuals_vs_fitted.png"), p1) + + p2 <- ggplot(residual_data, aes(sample = standardized)) + + stat_qq() + + stat_qq_line(color = "red") + + labs(title = "Normal Q-Q Plot") + + theme_minimal() + + ggsave(file.path(output_dir, "qq_plot.png"), p2) + + invisible(list(p1, p2)) +} \ No newline at end of file diff --git a/tests/data/header-files-test/cache.lua b/tests/data/header-files-test/cache.lua new file mode 100644 index 00000000..4f611936 --- /dev/null +++ b/tests/data/header-files-test/cache.lua @@ -0,0 +1,127 @@ +-- Copyright (c) 2024 OpenResty Inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +local require = require +local setmetatable = setmetatable +local ngx = ngx +local type = type +local error = error +local tostring = tostring +local math_floor = math.floor +local os_time = os.time + +local _M = {} +local mt = { __index = _M } + +_M._VERSION = '0.1.0' + +function _M.new(max_size, default_ttl) + if type(max_size) ~= "number" or max_size < 1 then + error("max_size must be a positive number") + end + + local self = { + store = {}, + expiry = {}, + max_size = max_size, + size = 0, + default_ttl = default_ttl or 300, + hits = 0, + misses = 0, + } + + return setmetatable(self, mt) +end + +function _M.set(self, key, value, ttl) + if key == nil then + return nil, "key is nil" + end + + ttl = ttl or self.default_ttl + + if self.store[key] == nil then + if self.size >= self.max_size then + self:_evict() + end + self.size = self.size + 1 + end + + self.store[key] = value + self.expiry[key] = os_time() + ttl + + return true +end + +function _M.get(self, key) + local value = self.store[key] + if value == nil then + self.misses = self.misses + 1 + return nil, "not found" + end + + local exp = self.expiry[key] + if exp and os_time() > exp then + self:delete(key) + self.misses = self.misses + 1 + return nil, "expired" + end + + self.hits = self.hits + 1 + return value +end + +function _M.delete(self, key) + if self.store[key] ~= nil then + self.store[key] = nil + self.expiry[key] = nil + self.size = self.size - 1 + return true + end + return false +end + +function _M._evict(self) + local oldest_key = nil + local oldest_time = nil + + for key, exp in pairs(self.expiry) do + if oldest_time == nil or exp < oldest_time then + oldest_key = key + oldest_time = exp + end + end + + if oldest_key then + self:delete(oldest_key) + end +end + +function _M.flush(self) + self.store = {} + self.expiry = {} + self.size = 0 +end + +function _M.stats(self) + return { + size = self.size, + max_size = self.max_size, + hits = self.hits, + misses = self.misses, + hit_rate = self.hits / (self.hits + self.misses + 1), + } +end + +return _M \ No newline at end of file diff --git a/tests/data/header-files-test/config.rs b/tests/data/header-files-test/config.rs new file mode 100644 index 00000000..4db5e7d7 --- /dev/null +++ b/tests/data/header-files-test/config.rs @@ -0,0 +1,103 @@ +// Copyright 2024 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::collections::HashMap; +use std::fs; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// Configuration for the application. +/// +/// This struct holds all the configuration parameters that can be +/// loaded from a TOML configuration file or set via environment +/// variables. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Config { + pub server: ServerConfig, + pub database: DatabaseConfig, + pub logging: LoggingConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServerConfig { + pub host: String, + pub port: u16, + pub workers: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DatabaseConfig { + pub url: String, + pub max_connections: u32, + pub min_connections: u32, + pub connection_timeout: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LoggingConfig { + pub level: String, + pub file: Option, + pub format: String, +} + +impl Config { + /// Load configuration from the specified file path. + pub fn from_file(path: &Path) -> io::Result { + let mut file = fs::File::open(path)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + toml::from_str(&contents) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) + } + + /// Load configuration with environment variable overrides. + pub fn from_env() -> Self { + let mut config = Self::default(); + + if let Ok(host) = std::env::var("SERVER_HOST") { + config.server.host = host; + } + if let Ok(port) = std::env::var("SERVER_PORT") { + if let Ok(port) = port.parse() { + config.server.port = port; + } + } + if let Ok(url) = std::env::var("DATABASE_URL") { + config.database.url = url; + } + + config + } +} + +impl Default for Config { + fn default() -> Self { + Config { + server: ServerConfig { + host: "127.0.0.1".to_string(), + port: 8080, + workers: num_cpus::get(), + }, + database: DatabaseConfig { + url: "postgres://localhost/mydb".to_string(), + max_connections: 10, + min_connections: 1, + connection_timeout: 30, + }, + logging: LoggingConfig { + level: "info".to_string(), + file: None, + format: "pretty".to_string(), + }, + } + } +} \ No newline at end of file diff --git a/tests/data/header-files-test/core.clj b/tests/data/header-files-test/core.clj new file mode 100644 index 00000000..3303ca0d --- /dev/null +++ b/tests/data/header-files-test/core.clj @@ -0,0 +1,66 @@ +;; Copyright (c) Rich Hickey. All rights reserved. +;; The use and distribution terms for this software are covered by the +;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) +;; which can be found in the file epl-v10.html at the root of this distribution. +;; By using this software in any fashion, you are agreeing to be bound by +;; the terms of this license. +;; You must not remove this notice, or any other, from this software. + +(ns myapp.core + (:require [clojure.string :as str] + [clojure.java.io :as io] + [clojure.edn :as edn] + [clojure.tools.logging :as log])) + +(defn load-config + "Load configuration from an EDN file. + Returns a map of configuration values." + [path] + (try + (with-open [reader (io/reader path)] + (edn/read (java.io.PushbackReader. reader))) + (catch Exception e + (log/error e "Failed to load config from" path) + {}))) + +(defn parse-request + "Parse an HTTP request string into a map." + [request-str] + (let [lines (str/split-lines request-str) + [method path version] (str/split (first lines) #"\s+") + headers (->> (rest lines) + (take-while (complement str/blank?)) + (map #(str/split % #":\s*" 2)) + (filter #(= 2 (count %))) + (into {} (map (fn [[k v]] [(str/lower-case k) v]))))] + {:method method + :path path + :version version + :headers headers})) + +(defn route-request + "Route a request to the appropriate handler." + [routes request] + (let [handler (get-in routes [(:method request) (:path request)])] + (if handler + (handler request) + {:status 404 + :body "Not Found"}))) + +(defn start-server + "Start the application server with the given configuration." + [config] + (let [port (get config :port 8080) + host (get config :host "0.0.0.0")] + (log/info "Starting server on" host ":" port) + {:port port + :host host + :status :running})) + +(defn -main + "Application entry point." + [& args] + (let [config-path (or (first args) "config.edn") + config (load-config config-path)] + (log/info "Loaded configuration:" config) + (start-server config))) \ No newline at end of file diff --git a/tests/data/header-files-test/crc32c.c b/tests/data/header-files-test/crc32c.c new file mode 100644 index 00000000..6edadd2f --- /dev/null +++ b/tests/data/header-files-test/crc32c.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: Zlib +/* crc32c.c -- compute CRC-32C using the Intel crc32 instruction + * Copyright (C) 2013 Mark Adler + * Version 1.1 1 Aug 2013 Mark Adler + */ + +/* + This software is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Mark Adler + madler@alumni.caltech.edu + */ + +/** + @file crc32c.c + @date 14 Dec 2020 + @brief Use hardware CRC instruction on Intel SSE 4.2 processors. + This computes a + CRC-32C, *not* the CRC-32 used by Ethernet and zip, gzip, etc. A software + version is provided as a fall-back, as well as for speed comparisons. + @see https://github.com/scanoss/engine/blob/master/external/src/crc32c.c + */ + +/* Version history: + 1.0 10 Feb 2013 First version + 1.1 1 Aug 2013 Correct comments on why three crc instructions in parallel + */ + +#include +#include +#include +#include +#include +#include + +/* CRC-32C (iSCSI) polynomial in reversed bit order. */ +#define POLY 0x82f63b78 + +/* Table for a quadword-at-a-time software crc. */ +static pthread_once_t crc32c_once_sw = PTHREAD_ONCE_INIT; +static uint32_t crc32c_table[8][256]; + +/** + * @brief Construct table for software CRC-32C calculation. + */ +static void crc32c_init_sw(void) +{ + uint32_t n, crc, k; + + for (n = 0; n < 256; n++) { + crc = n; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; + crc32c_table[0][n] = crc; + } + for (n = 0; n < 256; n++) { + crc = crc32c_table[0][n]; + for (k = 1; k < 8; k++) { + crc = crc32c_table[0][crc & 0xff] ^ (crc >> 8); + crc32c_table[k][n] = crc; + } + } +} + +/** + * @brief Table-driven software version as a fall-back. This is about 15 times slower + than using the hardware instructions. This assumes little-endian integers, + as is the case on Intel processors that the assembler code here is for. + * @param crci Acummulated valued + * @param buf Data buffer + * @param len Data buffer lenght + * @return CRC32 + */ +static uint32_t crc32c_sw(uint32_t crci, const void *buf, size_t len) +{ + const unsigned char *next = buf; + uint64_t crc; + + pthread_once(&crc32c_once_sw, crc32c_init_sw); + crc = crci ^ 0xffffffff; + while (len && ((uintptr_t)next & 7) != 0) { + crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + while (len >= 8) { + crc ^= *(uint64_t *)next; + crc = crc32c_table[7][crc & 0xff] ^ + crc32c_table[6][(crc >> 8) & 0xff] ^ + crc32c_table[5][(crc >> 16) & 0xff] ^ + crc32c_table[4][(crc >> 24) & 0xff] ^ + crc32c_table[3][(crc >> 32) & 0xff] ^ + crc32c_table[2][(crc >> 40) & 0xff] ^ + crc32c_table[1][(crc >> 48) & 0xff] ^ + crc32c_table[0][crc >> 56]; + next += 8; + len -= 8; + } + while (len) { + crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + return (uint32_t)crc ^ 0xffffffff; +} + +/** + * @brief Multiply a matrix times a vector over the Galois field of two elements, + GF(2). Each element is a bit in an unsigned integer. mat must have at + least as many entries as the power of two for most significant one bit in + vec. + * @param mat Input matrix + * @param vec Input vector + * @return result + */ +static inline uint32_t gf2_matrix_times(uint32_t *mat, uint32_t vec) +{ + uint32_t sum; + + sum = 0; + while (vec) { + if (vec & 1) + sum ^= *mat; + vec >>= 1; + mat++; + } + return sum; +} + +/** + * @brief Multiply a matrix by itself over GF(2). Both mat and square must have 32 + rows. + * @param square Output pointer + * @param mat Input matrix + */ +static inline void gf2_matrix_square(uint32_t *square, uint32_t *mat) +{ + int n; + + for (n = 0; n < 32; n++) + square[n] = gf2_matrix_times(mat, mat[n]); +} + +/** + * @brief Construct an operator to apply len zeros to a crc. len must be a power of + two. If len is not a power of two, then the result is the same as for the + largest power of two less than len. The result for len == 0 is the same as + for len == 1. A version of this routine could be easily written for any + len, but that is not needed for this application. + * @param even //TODO + * @param len //TODO + */ +static void crc32c_zeros_op(uint32_t *even, size_t len) +{ + int n; + uint32_t row; + uint32_t odd[32]; /* odd-power-of-two zeros operator */ + + /* put operator for one zero bit in odd */ + odd[0] = POLY; /* CRC-32C polynomial */ + row = 1; + for (n = 1; n < 32; n++) { + odd[n] = row; + row <<= 1; + } + + /* put operator for two zero bits in even */ + gf2_matrix_square(even, odd); + + /* put operator for four zero bits in odd */ + gf2_matrix_square(odd, even); + + /* first square will put the operator for one zero byte (eight zero bits), + in even -- next square puts operator for two zero bytes in odd, and so + on, until len has been rotated down to zero */ + do { + gf2_matrix_square(even, odd); + len >>= 1; + if (len == 0) + return; + gf2_matrix_square(odd, even); + len >>= 1; + } while (len); + + /* answer ended up in odd -- copy to even */ + for (n = 0; n < 32; n++) + even[n] = odd[n]; +} + +/** + * @brief Take a length and build four lookup tables for applying the zeros operator + for that length, byte-by-byte on the operand. + * @param zeros //TODO + * @param len //TODO + */ +static void crc32c_zeros(uint32_t zeros[][256], size_t len) +{ + uint32_t n; + uint32_t op[32]; + + crc32c_zeros_op(op, len); + for (n = 0; n < 256; n++) { + zeros[0][n] = gf2_matrix_times(op, n); + zeros[1][n] = gf2_matrix_times(op, n << 8); + zeros[2][n] = gf2_matrix_times(op, n << 16); + zeros[3][n] = gf2_matrix_times(op, n << 24); + } +} + +/** + * @brief Apply the zeros operator table to crc. + * @param zeros //TODO + * @param crc //TODO + * @return //TODO + */ +static inline uint32_t crc32c_shift(uint32_t zeros[][256], uint32_t crc) +{ + return zeros[0][crc & 0xff] ^ zeros[1][(crc >> 8) & 0xff] ^ + zeros[2][(crc >> 16) & 0xff] ^ zeros[3][crc >> 24]; +} + +/* Block sizes for three-way parallel crc computation. LONG and SHORT must + both be powers of two. The associated string constants must be set + accordingly, for use in constructing the assembler instructions. */ +#define LONG 8192 +#define LONGx1 "8192" +#define LONGx2 "16384" +#define SHORT 256 +#define SHORTx1 "256" +#define SHORTx2 "512" + +/* Tables for hardware crc that shift a crc by LONG and SHORT zeros. */ +static pthread_once_t crc32c_once_hw = PTHREAD_ONCE_INIT; +static uint32_t crc32c_long[4][256]; +static uint32_t crc32c_short[4][256]; + +/** + * @brief Initialize tables for shifting crcs. + */ +static void crc32c_init_hw(void) +{ + crc32c_zeros(crc32c_long, LONG); + crc32c_zeros(crc32c_short, SHORT); +} + +/** + * @brief Compute CRC-32C using the Intel hardware instruction. + * @param crc //TODO + * @param buf //TODO + * @param len //TODO + * @return //TODO + */ +static uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len) +{ + const unsigned char *next = buf; + const unsigned char *end; + uint64_t crc0, crc1, crc2; /* need to be 64 bits for crc32q */ + + /* populate shift tables the first time through */ + pthread_once(&crc32c_once_hw, crc32c_init_hw); + + /* pre-process the crc */ + crc0 = crc ^ 0xffffffff; + + /* compute the crc for up to seven leading bytes to bring the data pointer + to an eight-byte boundary */ + while (len && ((uintptr_t)next & 7) != 0) { + __asm__("crc32b\t" "(%1), %0" + : "=r"(crc0) + : "r"(next), "0"(crc0)); + next++; + len--; + } + + /* compute the crc on sets of LONG*3 bytes, executing three independent crc + instructions, each on LONG bytes -- this is optimized for the Nehalem, + Westmere, Sandy Bridge, and Ivy Bridge architectures, which have a + throughput of one crc per cycle, but a latency of three cycles */ + while (len >= LONG*3) { + crc1 = 0; + crc2 = 0; + end = next + LONG; + do { + __asm__("crc32q\t" "(%3), %0\n\t" + "crc32q\t" LONGx1 "(%3), %1\n\t" + "crc32q\t" LONGx2 "(%3), %2" + : "=r"(crc0), "=r"(crc1), "=r"(crc2) + : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2)); + next += 8; + } while (next < end); + crc0 = crc32c_shift(crc32c_long, crc0) ^ crc1; + crc0 = crc32c_shift(crc32c_long, crc0) ^ crc2; + next += LONG*2; + len -= LONG*3; + } + + /* do the same thing, but now on SHORT*3 blocks for the remaining data less + than a LONG*3 block */ + while (len >= SHORT*3) { + crc1 = 0; + crc2 = 0; + end = next + SHORT; + do { + __asm__("crc32q\t" "(%3), %0\n\t" + "crc32q\t" SHORTx1 "(%3), %1\n\t" + "crc32q\t" SHORTx2 "(%3), %2" + : "=r"(crc0), "=r"(crc1), "=r"(crc2) + : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2)); + next += 8; + } while (next < end); + crc0 = crc32c_shift(crc32c_short, crc0) ^ crc1; + crc0 = crc32c_shift(crc32c_short, crc0) ^ crc2; + next += SHORT*2; + len -= SHORT*3; + } + + /* compute the crc on the remaining eight-byte units less than a SHORT*3 + block */ + end = next + (len - (len & 7)); + while (next < end) { + __asm__("crc32q\t" "(%1), %0" + : "=r"(crc0) + : "r"(next), "0"(crc0)); + next += 8; + } + len &= 7; + + /* compute the crc for up to seven trailing bytes */ + while (len) { + __asm__("crc32b\t" "(%1), %0" + : "=r"(crc0) + : "r"(next), "0"(crc0)); + next++; + len--; + } + + /* return a post-processed crc */ + return (uint32_t)crc0 ^ 0xffffffff; +} + +/* Check for SSE 4.2. SSE 4.2 was first supported in Nehalem processors + introduced in November, 2008. This does not check for the existence of the + cpuid instruction itself, which was introduced on the 486SL in 1992, so this + will fail on earlier x86 processors. cpuid works on all Pentium and later + processors. */ +#ifdef __x86_64__ +#define SSE42(have) \ + do { \ + uint32_t eax, ecx; \ + eax = 1; \ + __asm__("cpuid" \ + : "=c"(ecx) \ + : "a"(eax) \ + : "%ebx", "%edx"); \ + (have) = (ecx >> 20) & 1; \ + } while (0) +#endif + +/** + * @brief Compute a CRC-32C. If the crc32 instruction is available, use the hardware + version. Otherwise, use the software version. + * @param crc //TODO + * @param buf //TODO + * @param len //TODO + * @return //TODO + */ +uint32_t crc32c(uint32_t crc, const void *buf, size_t len) +{ + int sse42; +#ifdef __x86_64__ + SSE42(sse42); + return sse42 ? crc32c_hw(crc, buf, len) : crc32c_sw(crc, buf, len); +#else + return crc32c_sw(crc, buf, len); +#endif +} + + +#define SIZE (262144*3) +#define CHUNK SIZE + +/** + * @brief //TODO + * @param data //TODO + * @param len //TODO + * @return //TODO + */ +uint32_t calc_crc32c (char *data, ssize_t len) { + + uint32_t crc = 0; + size_t off, n; + off = 0; + + do { + n = (size_t)len - off; + if (n > CHUNK) n = CHUNK; + crc = crc32c(crc, data + off, n); + off += n; + } while (off < (size_t)len); + + return crc; +} + diff --git a/tests/data/header-files-test/deploy.sh b/tests/data/header-files-test/deploy.sh new file mode 100644 index 00000000..44f22a65 --- /dev/null +++ b/tests/data/header-files-test/deploy.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Copyright (c) 2024 HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 +# +# This script deploys the application to the target environment. +# It handles building, testing, and deploying in a single step. + +set -euo pipefail + +# Configuration +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +readonly BUILD_DIR="${PROJECT_ROOT}/build" +readonly DEPLOY_ENV="${1:-staging}" +readonly VERSION="${2:-$(git describe --tags --always)}" +readonly TIMESTAMP="$(date -u +%Y%m%d%H%M%S)" + +# Colors for output +readonly RED='\033[0;31m' +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly NC='\033[0m' + +log_info() { + echo -e "${GREEN}[INFO]${NC} $*" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" >&2 +} + +check_prerequisites() { + local missing=() + + for cmd in docker kubectl jq; do + if ! command -v "$cmd" &> /dev/null; then + missing+=("$cmd") + fi + done + + if [ ${#missing[@]} -gt 0 ]; then + log_error "Missing required tools: ${missing[*]}" + exit 1 + fi + + log_info "All prerequisites met" +} + +build_image() { + local image_tag="myapp:${VERSION}" + + log_info "Building Docker image: ${image_tag}" + + docker build \ + --build-arg VERSION="${VERSION}" \ + --build-arg BUILD_DATE="${TIMESTAMP}" \ + --tag "${image_tag}" \ + --file "${PROJECT_ROOT}/Dockerfile" \ + "${PROJECT_ROOT}" + + log_info "Image built successfully: ${image_tag}" + echo "${image_tag}" +} + +run_tests() { + log_info "Running test suite..." + + if ! docker run --rm "myapp:${VERSION}" test; then + log_error "Tests failed" + exit 1 + fi + + log_info "All tests passed" +} + +deploy() { + local env="$1" + local image_tag="$2" + + log_info "Deploying ${image_tag} to ${env}" + + kubectl set image "deployment/myapp" \ + "myapp=${image_tag}" \ + --namespace="${env}" \ + --record + + kubectl rollout status "deployment/myapp" \ + --namespace="${env}" \ + --timeout=300s + + log_info "Deployment to ${env} completed successfully" +} + +main() { + log_info "Starting deployment pipeline" + log_info "Environment: ${DEPLOY_ENV}" + log_info "Version: ${VERSION}" + + check_prerequisites + + local image_tag + image_tag=$(build_image) + + run_tests + + deploy "${DEPLOY_ENV}" "${image_tag}" + + log_info "Pipeline completed successfully" +} + +main "$@" \ No newline at end of file diff --git a/tests/data/header-files-test/handler.go b/tests/data/header-files-test/handler.go new file mode 100644 index 00000000..2c65d8ef --- /dev/null +++ b/tests/data/header-files-test/handler.go @@ -0,0 +1,84 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" +) + +// Handler responds to an HTTP request. +type Handler interface { + ServeHTTP(w http.ResponseWriter, r *http.Request) +} + +// Router is an HTTP request multiplexer that matches the URL +// of each incoming request against a list of registered patterns. +type Router struct { + mu sync.RWMutex + routes map[string]Handler + notFound Handler +} + +// NewRouter creates a new Router instance. +func NewRouter() *Router { + return &Router{ + routes: make(map[string]Handler), + } +} + +// Handle registers the handler for the given pattern. +func (r *Router) Handle(pattern string, handler Handler) { + r.mu.Lock() + defer r.mu.Unlock() + r.routes[pattern] = handler +} + +// HandleFunc registers the handler function for the given pattern. +func (r *Router) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) { + r.Handle(pattern, http.HandlerFunc(handler)) +} + +// ServeHTTP dispatches the request to the handler whose +// pattern most closely matches the request URL. +func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { + r.mu.RLock() + handler, ok := r.routes[req.URL.Path] + r.mu.RUnlock() + + if !ok { + if r.notFound != nil { + r.notFound.ServeHTTP(w, req) + return + } + http.NotFound(w, req) + return + } + + handler.ServeHTTP(w, req) +} + +// JSONResponse writes a JSON response with the given status code. +func JSONResponse(w http.ResponseWriter, statusCode int, data interface{}) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + return json.NewEncoder(w).Encode(data) +} + +// ReadJSON reads a JSON request body into the given destination. +func ReadJSON(r *http.Request, dst interface{}) error { + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + return fmt.Errorf("reading body: %w", err) + } + return json.Unmarshal(body, dst) +} \ No newline at end of file diff --git a/tests/data/header-files-test/logger.rb b/tests/data/header-files-test/logger.rb new file mode 100644 index 00000000..ad423ac3 --- /dev/null +++ b/tests/data/header-files-test/logger.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Copyright (c) 2024 Rails Contributors +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + +require 'logger' +require 'fileutils' +require 'singleton' + +module ActiveSupport + class TaggedLogging + include Singleton + + SEVERITIES = %i[debug info warn error fatal unknown].freeze + DEFAULT_FORMAT = "%s [%s] %s -- %s: %s\n" + + attr_reader :logger, :tags + + def initialize + @logger = ::Logger.new($stdout) + @logger.formatter = method(:default_formatter) + @tags = [] + @mutex = Mutex.new + end + + def tagged(*new_tags, &block) + @mutex.synchronize do + @tags.concat(new_tags.flatten) + result = block.call(self) + @tags.pop(new_tags.flatten.size) + result + end + end + + SEVERITIES.each do |severity| + define_method(severity) do |message = nil, &block| + message = block.call if block + return if message.nil? + + formatted_tags = @tags.map { |t| "[#{t}]" }.join(" ") + @logger.send(severity, "#{formatted_tags} #{message}".strip) + end + end + + def silence(temporary_level = ::Logger::ERROR, &block) + old_level = @logger.level + @logger.level = temporary_level + yield self + ensure + @logger.level = old_level + end + + private + + def default_formatter(severity, datetime, progname, msg) + format(DEFAULT_FORMAT, + severity[0], + datetime.strftime("%Y-%m-%dT%H:%M:%S.%6N"), + $$, + progname, + msg) + end + end +end \ No newline at end of file diff --git a/tests/data/header-files-test/multiline_imports.py b/tests/data/header-files-test/multiline_imports.py new file mode 100644 index 00000000..8e22d972 --- /dev/null +++ b/tests/data/header-files-test/multiline_imports.py @@ -0,0 +1,99 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2021, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import argparse +import os +import sys +import traceback +from dataclasses import asdict +from pathlib import Path +from typing import List + +import pypac + +from scanoss.cryptography import Cryptography, create_cryptography_config_from_args +from scanoss.delta import Delta +from scanoss.export.dependency_track import DependencyTrackExporter +from scanoss.scanners.container_scanner import ( + DEFAULT_SYFT_COMMAND, + DEFAULT_SYFT_TIMEOUT, + ContainerScanner, + create_container_scanner_config_from_args, +) +from scanoss.scanners.folder_hasher import ( + FolderHasher, + create_folder_hasher_config_from_args, +) +from scanoss.scanossgrpc import ( + ScanossGrpc, + ScanossGrpcError, + create_grpc_config_from_args, +) + +from .components import Components +from .constants import ( + DEFAULT_API_TIMEOUT, + DEFAULT_COPYLEFT_LICENSE_SOURCES, + DEFAULT_HFH_DEPTH, + DEFAULT_HFH_MIN_ACCEPTED_SCORE, + DEFAULT_HFH_RANK_THRESHOLD, + DEFAULT_HFH_RECURSIVE_THRESHOLD, + DEFAULT_POST_SIZE, + DEFAULT_RETRY, + DEFAULT_TIMEOUT, + MIN_TIMEOUT, + PYTHON_MAJOR_VERSION, + VALID_LICENSE_SOURCES, +) +from .csvoutput import CsvOutput +from .cyclonedx import CycloneDx +from .filecount import FileCount +from .gitlabqualityreport import GitLabQualityReport +from .inspection.policy_check.dependency_track.project_violation import ( + DependencyTrackProjectViolationPolicyCheck, +) +from .inspection.policy_check.scanoss.copyleft import Copyleft +from .inspection.policy_check.scanoss.undeclared_component import UndeclaredComponent +from .inspection.summary.component_summary import ComponentSummary +from .inspection.summary.license_summary import LicenseSummary +from .inspection.summary.match_summary import MatchSummary +from .results import Results +from .scancodedeps import ScancodeDeps +from .scanner import FAST_WINNOWING, Scanner +from .scanners.scanner_config import create_scanner_config_from_args +from .scanners.scanner_hfh import ScannerHFH +from .scanoss_settings import ScanossSettings, ScanossSettingsError +from .scantype import ScanType +from .spdxlite import SpdxLite +from .threadeddependencies import SCOPE +from .utils.file import validate_json_file + +HEADER_PARTS_COUNT = 2 + + +def print_stderr(*args, **kwargs): + """ + Print the given message to STDERR + """ + print(*args, file=sys.stderr, **kwargs) \ No newline at end of file diff --git a/tests/data/header-files-test/parser.pl b/tests/data/header-files-test/parser.pl new file mode 100644 index 00000000..097090e1 --- /dev/null +++ b/tests/data/header-files-test/parser.pl @@ -0,0 +1,94 @@ +#!/usr/bin/perl +# Copyright (c) 2024 The Perl Foundation +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the Artistic License 2.0. +# +# See https://opensource.org/licenses/Artistic-2.0 +# for the full license text. + +use strict; +use warnings; +use Getopt::Long; +use File::Basename; +use Carp qw(croak); + +my $VERSION = '1.0.0'; + +sub new { + my ($class, %args) = @_; + my $self = bless { + delimiter => $args{delimiter} || ',', + quote_char => $args{quote_char} || '"', + escape => $args{escape} || '\\', + headers => $args{headers} || [], + strict => $args{strict} // 1, + line_num => 0, + }, $class; + return $self; +} + +sub parse_file { + my ($self, $filename) = @_; + croak "Filename required" unless defined $filename; + + open my $fh, '<:encoding(UTF-8)', $filename + or croak "Cannot open '$filename': $!"; + + my @records; + my $header_line = <$fh>; + chomp $header_line; + $self->{headers} = $self->_split_line($header_line); + $self->{line_num} = 1; + + while (my $line = <$fh>) { + chomp $line; + $self->{line_num}++; + next if $line =~ /^\s*$/; + next if $line =~ /^\s*#/; + + my $fields = $self->_split_line($line); + if ($self->{strict} && scalar @$fields != scalar @{$self->{headers}}) { + croak sprintf( + "Field count mismatch at line %d: expected %d, got %d", + $self->{line_num}, + scalar @{$self->{headers}}, + scalar @$fields + ); + } + + my %record; + for my $i (0 .. $#{$self->{headers}}) { + $record{$self->{headers}[$i]} = $fields->[$i] // ''; + } + push @records, \%record; + } + + close $fh; + return \@records; +} + +sub _split_line { + my ($self, $line) = @_; + my @fields; + my $field = ''; + my $in_quotes = 0; + + for my $char (split //, $line) { + if ($char eq $self->{quote_char} && !$in_quotes) { + $in_quotes = 1; + } elsif ($char eq $self->{quote_char} && $in_quotes) { + $in_quotes = 0; + } elsif ($char eq $self->{delimiter} && !$in_quotes) { + push @fields, $field; + $field = ''; + } else { + $field .= $char; + } + } + push @fields, $field; + + return \@fields; +} + +1; \ No newline at end of file diff --git a/tests/data/header-files-test/results.py b/tests/data/header-files-test/results.py new file mode 100644 index 00000000..5cbff282 --- /dev/null +++ b/tests/data/header-files-test/results.py @@ -0,0 +1,275 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2024, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import json +from typing import Any, Dict, List + +from scanoss.utils.abstract_presenter import AbstractPresenter + +from .scanossbase import ScanossBase + +MATCH_TYPES = ['file', 'snippet'] +STATUSES = ['pending', 'identified'] + + +AVAILABLE_FILTER_VALUES = { + 'match_type': [e for e in MATCH_TYPES], + 'status': [e for e in STATUSES], +} + + +ARG_TO_FILTER_MAP = { + 'match_type': 'id', + 'status': 'status', +} + +PENDING_IDENTIFICATION_FILTERS = { + 'match_type': ['file', 'snippet'], + 'status': ['pending'], +} + + +class ResultsPresenter(AbstractPresenter): + """ + SCANOSS Results presenter class + Handles the presentation of the scan results + """ + + def __init__(self, results_instance, **kwargs): + super().__init__(**kwargs) + self.results = results_instance + + def _format_json_output(self) -> str: + """ + Format the output data into a JSON object + """ + + formatted_data = [] + for item in self.results.data: + formatted_data.append( + { + 'file': item.get('filename'), + 'status': item.get('status', 'N/A'), + 'match_type': item['id'], + 'matched': item.get('matched', 'N/A'), + 'purl': (item.get('purl')[0] if item.get('purl') else 'N/A'), + 'license': (item.get('licenses')[0].get('name', 'N/A') if item.get('licenses') else 'N/A'), + } + ) + try: + return json.dumps({'results': formatted_data, 'total': len(formatted_data)}, indent=2) + except Exception as e: + self.base.print_stderr(f'ERROR: Problem formatting JSON output: {e}') + return '' + + def _format_cyclonedx_output(self) -> str: + raise NotImplementedError('CycloneDX output is not implemented') + + def _format_spdxlite_output(self) -> str: + raise NotImplementedError('SPDXlite output is not implemented') + + def _format_csv_output(self) -> str: + raise NotImplementedError('CSV output is not implemented') + + def _format_raw_output(self) -> str: + raise NotImplementedError('Raw output is not implemented') + + def _format_plain_output(self) -> str: + """Format the output data into a plain text string + + Returns: + str: The formatted output data + """ + if not self.results.data: + msg = 'No results to present' + return msg + + formatted = '' + for item in self.results.data: + formatted += f'{self._format_plain_output_item(item)}\n' + return formatted + + @staticmethod + def _format_plain_output_item(item): + purls = item.get('purl', []) + licenses = item.get('licenses', []) + + return ( + f'File: {item.get("filename")}\n' + f'Match type: {item.get("id")}\n' + f'Status: {item.get("status", "N/A")}\n' + f'Matched: {item.get("matched", "N/A")}\n' + f'Purl: {purls[0] if purls else "N/A"}\n' + f'License: {licenses[0].get("name", "N/A") if licenses else "N/A"}\n' + ) + + +class Results: + """ + SCANOSS Results class \n + Handles the parsing and filtering of the scan results + """ + + def __init__( # noqa: PLR0913 + self, + debug: bool = False, + trace: bool = False, + quiet: bool = False, + filepath: str = None, + match_type: str = None, + status: str = None, + output_file: str = None, + output_format: str = None, + ): + """Initialise the Results class + + Args: + debug (bool, optional): Debug. Defaults to False. + trace (bool, optional): Trace. Defaults to False. + quiet (bool, optional): Quiet. Defaults to False. + filepath (str, optional): Path to the scan results file. Defaults to None. + match_type (str, optional): Comma separated match type filters. Defaults to None. + status (str, optional): Comma separated status filters. Defaults to None. + output_file (str, optional): Path to the output file. Defaults to None. + output_format (str, optional): Output format. Defaults to None. + """ + + self.base = ScanossBase(debug, trace, quiet) + self.data = self._load_and_transform(filepath) + self.filters = self._load_filters(match_type=match_type, status=status) + self.presenter = ResultsPresenter( + self, + debug=debug, + trace=trace, + quiet=quiet, + output_file=output_file, + output_format=output_format, + ) + + def load_file(self, file: str) -> Dict[str, Any]: + """Load the JSON file + + Args: + file (str): Path to the JSON file + + Returns: + Dict[str, Any]: The parsed JSON data + """ + with open(file, 'r') as jsonfile: + try: + return json.load(jsonfile) + except Exception as e: + self.base.print_stderr(f'ERROR: Problem parsing input JSON: {e}') + + def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: + """ + Load the file and transform the data into a list of dictionaries with the filename and the file data + """ + + raw_data = self.load_file(file) + return self._transform_data(raw_data) + + @staticmethod + def _transform_data(data: dict) -> list: + """Transform the data into a list of dictionaries with the filename and the file data + + Args: + data (dict): The raw data + + Returns: + list: The transformed data + """ + result = [] + for filename, file_data in data.items(): + if file_data: + file_obj = {'filename': filename} + file_obj.update(file_data[0]) + result.append(file_obj) + return result + + def _load_filters(self, **kwargs): + """Extract and parse the filters + + Returns: + dict: Parsed filters + """ + filters = {} + + for key, value in kwargs.items(): + if value: + filters[key] = self._extract_comma_separated_values(value) + + return filters + + @staticmethod + def _extract_comma_separated_values(values: str): + return [value.strip() for value in values.split(',')] + + def apply_filters(self): + """Apply the filters to the data""" + filtered_data = [] + for item in self.data: + if self._item_matches_filters(item): + filtered_data.append(item) + self.data = filtered_data + + return self + + def _item_matches_filters(self, item): + for filter_key, filter_values in self.filters.items(): + if not filter_values: + continue + + self._validate_filter_values(filter_key, filter_values) + + item_value = item.get(ARG_TO_FILTER_MAP[filter_key]) + if isinstance(filter_values, list): + if item_value not in filter_values: + return False + elif item_value != filter_values: + return False + return True + + @staticmethod + def _validate_filter_values(filter_key: str, filter_value: List[str]): + if any(value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) for value in filter_value): + valid_values = ', '.join(AVAILABLE_FILTER_VALUES.get(filter_key, [])) + raise ValueError( + f"ERROR: Invalid filter value '{filter_value}' for filter '{filter_key}'. " + f'Valid values are: {valid_values}' + ) + + def get_pending_identifications(self): + """Get files with 'pending' status and 'file' or 'snippet' match type""" + self.filters = PENDING_IDENTIFICATION_FILTERS + self.apply_filters() + + return self + + def has_results(self): + return bool(self.data) + + def present(self, output_format: str = None, output_file: str = None): + """Present the results in the selected format""" + self.presenter.present(output_format=output_format, output_file=output_file) diff --git a/tests/data/header-files-test/server.ex b/tests/data/header-files-test/server.ex new file mode 100644 index 00000000..c75a2e40 --- /dev/null +++ b/tests/data/header-files-test/server.ex @@ -0,0 +1,100 @@ +# Copyright (c) 2024 Plataformatec +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule MyApp.Server do + use GenServer + require Logger + alias MyApp.Config + import MyApp.Utils, only: [format_timestamp: 1] + + @moduledoc """ + A GenServer that manages TCP connections and handles + incoming requests from clients. + """ + + @default_port 4000 + @max_connections 100 + @timeout 30_000 + + defstruct [:socket, :port, :connections, :started_at] + + def start_link(opts \\ []) do + port = Keyword.get(opts, :port, @default_port) + GenServer.start_link(__MODULE__, port, name: __MODULE__) + end + + @impl true + def init(port) do + Logger.info("Starting server on port #{port}") + + case :gen_tcp.listen(port, [:binary, active: false, reuseaddr: true]) do + {:ok, socket} -> + state = %__MODULE__{ + socket: socket, + port: port, + connections: %{}, + started_at: DateTime.utc_now() + } + + {:ok, state, {:continue, :accept}} + + {:error, reason} -> + Logger.error("Failed to listen on port #{port}: #{inspect(reason)}") + {:stop, reason} + end + end + + @impl true + def handle_continue(:accept, state) do + case :gen_tcp.accept(state.socket, @timeout) do + {:ok, client} -> + Logger.debug("New connection accepted") + {:ok, pid} = Task.start(fn -> handle_client(client) end) + :gen_tcp.controlling_process(client, pid) + + connections = Map.put(state.connections, pid, client) + {:noreply, %{state | connections: connections}, {:continue, :accept}} + + {:error, :timeout} -> + {:noreply, state, {:continue, :accept}} + + {:error, reason} -> + Logger.error("Accept error: #{inspect(reason)}") + {:stop, reason, state} + end + end + + @impl true + def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do + connections = Map.delete(state.connections, pid) + {:noreply, %{state | connections: connections}} + end + + defp handle_client(socket) do + case :gen_tcp.recv(socket, 0) do + {:ok, data} -> + response = process_request(data) + :gen_tcp.send(socket, response) + handle_client(socket) + + {:error, :closed} -> + Logger.debug("Client disconnected") + :ok + end + end + + defp process_request(data) do + "HTTP/1.1 200 OK\r\nContent-Length: #{byte_size(data)}\r\n\r\n#{data}" + end +end \ No newline at end of file diff --git a/tests/data/header-files-test/server.js b/tests/data/header-files-test/server.js new file mode 100644 index 00000000..94bc11f1 --- /dev/null +++ b/tests/data/header-files-test/server.js @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Express Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +'use strict'; + +const http = require('http'); +const path = require('path'); +const fs = require('fs'); + +const DEFAULT_PORT = 3000; +const DEFAULT_HOST = '127.0.0.1'; + +class Server { + constructor(options = {}) { + this.port = options.port || DEFAULT_PORT; + this.host = options.host || DEFAULT_HOST; + this.routes = new Map(); + this.middleware = []; + } + + use(fn) { + if (typeof fn !== 'function') { + throw new TypeError('Middleware must be a function'); + } + this.middleware.push(fn); + return this; + } + + get(path, handler) { + this.routes.set(`GET:${path}`, handler); + return this; + } + + post(path, handler) { + this.routes.set(`POST:${path}`, handler); + return this; + } + + listen(callback) { + this.server = http.createServer((req, res) => { + this._handleRequest(req, res); + }); + + this.server.listen(this.port, this.host, () => { + if (callback) callback(this.port, this.host); + }); + + return this; + } + + _handleRequest(req, res) { + const key = `${req.method}:${req.url}`; + const handler = this.routes.get(key); + + if (handler) { + handler(req, res); + } else { + res.statusCode = 404; + res.end('Not Found'); + } + } + + close() { + if (this.server) { + this.server.close(); + } + } +} + +module.exports = Server; \ No newline at end of file diff --git a/tests/data/header-files-test/widget.dart b/tests/data/header-files-test/widget.dart new file mode 100644 index 00000000..6a5fb0bf --- /dev/null +++ b/tests/data/header-files-test/widget.dart @@ -0,0 +1,77 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter/foundation.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/widgets.dart'; + +/// A material design card widget. +/// +/// A card is a sheet of material used to represent some related information, +/// for example an album, a geographical location, a meal, contact details, etc. +class MaterialCard extends StatelessWidget { + const MaterialCard({ + super.key, + this.color, + this.elevation = 1.0, + this.shape, + this.borderOnForeground = true, + this.margin, + this.clipBehavior = Clip.none, + this.child, + }); + + final Color? color; + final double elevation; + final ShapeBorder? shape; + final bool borderOnForeground; + final EdgeInsetsGeometry? margin; + final Clip clipBehavior; + final Widget? child; + + @override + Widget build(BuildContext context) { + final CardThemeData cardTheme = CardTheme.of(context); + final CardThemeData defaults = _CardDefaults(context); + + return Semantics( + container: true, + child: Container( + margin: margin ?? cardTheme.margin ?? defaults.margin, + child: Material( + type: MaterialType.card, + color: color ?? cardTheme.color ?? defaults.color, + elevation: elevation, + shape: shape ?? cardTheme.shape ?? defaults.shape, + borderOnForeground: borderOnForeground, + clipBehavior: clipBehavior, + child: Semantics( + explicitChildNodes: true, + child: child, + ), + ), + ), + ); + } +} + +class _CardDefaults extends CardThemeData { + _CardDefaults(this.context); + + final BuildContext context; + + @override + Color? get color => Theme.of(context).cardColor; + + @override + double? get elevation => 1.0; + + @override + EdgeInsetsGeometry? get margin => const EdgeInsets.all(4.0); + + @override + ShapeBorder? get shape => const RoundedRectangleBorder( + borderRadius: BorderRadius.all(Radius.circular(12.0)), + ); +} \ No newline at end of file diff --git a/tests/data/test_src_files.tar.gz b/tests/data/test_src_files.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..682c9262d953ce5551cbe481257d0fc89c12b268 GIT binary patch literal 29169 zcmV(>K-j+@iwFR=hOub?1MGckW7{^e=>1y#3RFpIDwXL+9_^81H;OGM+VyKCx!qpJ zN2MjoW+Ra*N!jtbzQ6s>0{{V%@*}VAo)hghmhc!127|$1Fc>89VCnyS=Fjr-@(<6R z3Htk-|1PgS<-hVHR-Uaqd-`Pg$;y)zvAnYK!_z1KM?Cx7h~a0Jq+SfDc&D@ErgBhh zG`b;Q_Cx>u5BVf8{%2k|9Q(0*8hxHNfF94EKAk)MPnVxO;qhO6zWl?JXE6RxeprR^ zU;aGk*q{I9KjG|dM6hR1yzSwDRw)VO$F$%`MTd7RF!8`BN7iVcYtyF@^G>TKN} zr&ajshtO~ko-X~KL?I@OCX^mT_;2W^OIRDfCut1j68d@>4)AXpO#Eu4Rzc+6@I=6V zPNQ&`h%^#_;+tOY0Mh=L`AOLM18WjevCRj#SoPBcpqR+4W_OlIRWn2!DTkWCbuc8 zH!+q=AbH|VkC>Xrk`|S3!YJLQp;Rf{y*>Xzf*-`bht)k1`WL#}fTI@jg`iY^iv zCssJrBoIdy5Zqc_gHg>AvhHN&?nHxkPOb9I&>ta7VbWnZmWi>;=pRXN$6{SUvT)u; zEau=1S!oc3_v`>doZAVkU0XljV(4 zst^!cVOs)?hvIMD{XInmae-vOd5Nh~{9$sCo*AqK^E90$B;_TIO(63~%e6LXX$aH0 z7<#Ff$<3?ln(*T|qDg+?zVW>wPJKW-b*h^v7K%T_;vxwD z$rM&SvDw*Ne6qSJW(f)y6pC#SZ(}Y?Ivr|~AQ~&;v9ZH&HfylUSa~9Lz4)DIlD-24 zeg@?RRhbocMTiyHdz^A9yVw#4>c_9>F^;g05#qp)CqY8p#4QrG4j9u8<1z|; z{5GSFk=Z%b;B*#GVQHuX9MJQ}GZO1C785@n1m0K}D`QfJ8Vq<$`gKabLFo@r-8;e2 z3sL|yihaSbu^5cKV8Xi^%)E%B)F%WOJakHvPqar=ra&Kt1h{y}14H0A z?7V;0OA1I1L1C+v%!VaMt5X89g z6=D^)#Jq`;+un2L-h?{{ou5M2LQ+uUcTmg&glB^@9v6d1+@wt@EPUdtIaMSD)Oq6a z3;-Z7X`zuzMfBL8z|MP!obE{!i7%0ZB(9nXNNcIsGYnwimzUmnHVFfFfItRku0Ncq z*@x5bC2q~NP{0JxqkOUQRBZVJv>_~G+DRPvBSBVzGcO(zgMiR5MT3ClF>L#Z-v0`wb#Y4n`so0` zHZAUvV`oVb57obsHy$sZ04LS)_k|B~0si8AOr|~*LZoo=N4STE{w6*fDU!z0(kV!v z*@+89mXd)NMoF^dhcJ--(#beFS(=FnE^B;%N9Fhbl3-k**?~fUm z2P|-W@ghlwpo*St*h#QR6w*M|wbN!HbYp0zO%Vfcn6HdGQcOc4s8z-Zx|`j0O-!Tl zWf)CR;sKtqj~&M`2N@T|pxe8M@dHq)f!5++f9F@R{Qk$)(eo!KKm14ldzly)f6Tn$ z1yoz~(nW6(?Le6*1F{e-+)o4pAyMr|;lS^wlK1&nasgZSgWj7?tGU(R-`i~U+k5Ta znyNViQaoAB>ZFLk(T~SRtIwVvugNL(SWOq$@JJI(su9&VDPq919L&aY-(etm9tA_y zOb`Yspyps&t#U<@Mxz2dbtqOB@75}mfvq?pbYpo19tmKIQVsAL#))qe>3SDLM45z^>0xtp>s`JtS%296XU#AY0E@$z4H{}{sa3$1 z{H|Po&@g*~+T_YJfnoxdbI_1z!9@KfG?ZYy)R{6>!SRx0hPB5*ngSAjI1HdNngvdw zebns`fq)HK0}mBRwLoSIBWp~gq9%zyIYBcS+1+8yBV`!K?^1_Rj8Bj(foOupGeMaF z!txwecT|COiIdq#Y@tyGzKmEOmpNmgk&%PHL5ZgV(tH3i2^L;~i*+W5@-h$PkeL*K z_(3_x0_!or1|2l`7YbHrKuIeuG7FR76i*TcKv`JvVa`%fs?cJu$`Ipx4`XPJwpi3D z`2qjzyd`I|o~AVhXU>I!B;Dn+DLmnuc%Zetm$K0VX*vLvRp22Y398rLI6}b`;7%HM zjTf-p9{Z$?lGJ;>?X1>7O1AJEOXiT&CCv`EcVQ+8q`Y)-x03R9>dZR_aY*~hvG3uL z7wpAo>mLGQh!e}yDN1dH0WMQ?oPcnH5$v9!QQU}go4h#~n4%QhYlpBEksUU&prp*= z&YLn4wst@&Q$WJz5EWD{6i-L1eFoVl8fN11F(lWsf&jdr&B2-?MIU^lwcF$#tYL6Z zFrp;X9BLNlVqFv=&|bzTSaM@y-BcqW|NHMtSckB<1CPM=EJzc7JYx1xBmXK&&xpRb zWlKOaSLOs7+9#{nR2*F-=5WYj`?D13K8?t}e)DkUM}Sr!gZd28AU`Jg+bfmAuL6-h^#y+WIHrJ5;g6CS$Z&<&*BM!ip;OmvCAmmu(%Se01L}w zacpQAxf8b#s=2IHwfi)-;Ibm%EVPW!^I*lE2Zjv7hG-h-&MGJ`UT8Si^ZTH>jC~+` z1&x(NrZ}0I{LpA^Z9dFVG>dJZ24#vcA+t`+8Zbv>ijhKz?8ws@E9!Gb;4%gI?2}ns zw_K|$f~bBiF0#HL0Rzlr9yu=2M#iYKtucudvD zgW$+s4Oer8Zm>P59J+7j_|b_V-@XwCujw@AODBLTpY@ z?>vz^gG8z?Qep9uB$7u#Y?Wr4goieCrCB9Df$;`|D8@56JWoTM@T3Za&Oj+{YN}^8 z8{0r+n5_xZA6Ncs)k=B4BC42JwNqC`b>;b!AD?E)+&>sTI|b`QVy+z4f3^ZZwQZb9 zs?AtPND@TECahed4xb|@jn*9v=4baan2)3I6b%DxFP~>&I*Z~oE0vpR?J!6|Ujc?O zMKJYt)tOcfKgd%zHPl2sa3g zJfY8QQ{Jbus3jMn(^;FGh=^IT5j2<3*#dpVn?JE-?p=;y2{Z) z)g!4g#U3W4yNs1@7ioRi@mMhndD!XwFn`#SnY2w-B2LhE(2Fk>LLXx1?DxH7((gM} zqQJ@9v?{93qm^1+JX)^i3}ZqxM0Gu`;#|mSf-)2GuqmirU6#d+C6_H*b8J}ihz6Px zNNAGlI5i(XVRC_1`Q9JQq*TUZ&;T^WDXNPh@mQvWXF|HpZq@*jgdfu#G|E&S5kynq z9AJL6xAeuHf98$-iB{=al)gsf%iY1H-Uso~+s}ex&7^}n%toJjN_X5dVv&g@k8Gl)Gv(_JK8w3hA{9)Br zi)LzKj8%IJwy5|eKLdwRX3>F}fzXi`L)hKJ%%PPJXCbO^+KhyLD;`Ksn2jX5(0Y_{ zI{NeI*-NTlDf>#$AH}}}<8v~wR>khc!CF=OpUuKjJGwUy7weTUF|$XJizVCKVIi>@ z)mTYqAu1c0$6D%gaA1e}iG`=sjuJ5hH6*_oBE!-7r)Wd z2Gd@c&EdZNn{0SUV*k~*Gd}+ZeUJHMHgTtypQnwR@Bgf>E<>h$|L5uQlc!(r|9pv0 zwOXyry#Tb^?%|9!uI~(L6eOT7x|_|t{cg8-u?5`_PcGH{mO9-sK}7?F1{yo~!VB6e z5Sni}0L}&YF?Y-CH71Z3@?}?o@a5RAJ?NgHy zSi99laJ#L}<{S9geAV7*_kKmI-FCaThj6#|JEAELnw?&I^Khrx5eJ8zgMCojn|oUT zc(1*;-GO#myRE&R3++OhX#EU7U?#-Bw*RJMAu#WxKP#i`_)RLY;jA12y+r3PHX4TkX)F;a2|0v9l}hvHW_t%muD{*h>9sojpPM^}REt&41qxHp z$1ocU4ROSmI_PxJK;f8;#0z|XJy%X40);xA+_P<9Q?URfQ*%wl10sX4cz#lBei2xrFeO{HOKo9(EyTN;|q z;xW{7vWk>aL$^BE!sL?$ZVGRPDD?&#s7y=6(kS(iDloc^omsoC)45xit{aAF|NX$9rlLiEW=l!?_2@lcXuRe(+^ zwq|`3YN>RG*^8aIMfy&XQ;^&Xdy!gQXv4q@;6fR1Lxqwh0AVbc$@iB2XjmC7Tv72d zO7{^UNw3=V+c0ZOg{x6;7`d11cQWx|6E?hY1P!n8ps0I<7ruH0&wY+|Kvku5b--boy z&x^@h3OB7V7@#HMuls3%LOg)E_lE~d$WCYZzgF}K9?gbq%^dnCv(tt+iJ~zu(Y80{ z$Q=AZmDZn^`Nu5q(|MW5LsKt3YoOSG>^)dDvfNCKDbG(N+mz+WDKjFjy{Q1rvtp5& zB_fqSowWU1LzB&%CaA==%pkG}2RmOUcbO>k#-p49ibYmbTc*NI(HX3Slm~HDsOmk_D!&?ZI0Fwfs>7$ze8MDXLhPH zU*vXfa$Cep+BouteGn>rs1PQIgQK`1S4|Sot|=3pY^c|>uPQj+I{(SR%eK4g#l3Su zA+8sv+B$c&IE&V~^Ti3WPXE^D4fwk8y}S?&(Yp03D`@CsiZ;I7d27i;IvHAnj))7I z;pMR5Au@`CdSnVc>qq#K4naJrB}KEX1(? zf}ITRny9b2LCNj0@MG??kX(+cYj0iduwTN@UH43C84c*-Bz98mm$K!uWJ;8cB3ko~ zYziKi8#MHw(s+^CSwYSmTBQcAV%ZMabnnsqULB}XsbSJ~Eg=?$$=F|9^v&s>@uf{^ zbqGuoTO&tR9R;IyqY9g=L>e`9f@p=kvS`!$^aQwRWk&c8VZ%%geaclTogaiYRnXnT z6PkImLS6waSez8n%t3wGTY}@FI-58(yIs&Lb;%;HQ&x2HJ6bR^yHZ0pb3M0)JDXxG zD@Owdy|(CA#NAL9FIBg|Eb`uWZr-Q`foDvKeGv%OdQWyy8bcB{Y9z7dEklBXh{BG9 zl&VrT7};`lb#3ho#XJA9&S64u6d1v|<3Qy~#{i)p0jy)aBKq8~K8B+5`dS0&6O6>6 zKgGS5)$cj8>n@O4H#T}8;U7`1i~IShQzHuK&T~aKnT`Y4B-R&d$Juz&9X@M3?NQ7w zcqvlL)dv}?I1+cW#}+vr#(=4d<@z!vvPLVnK`w1laWa)HRfVkLg#p5n2Ah}S!R(&H zM$UaG&?lag-+95SB@ss#2T$$)zFBr2a%Y&w`7hF+voE_{8=IX_p|3;-~ag{pT$M-X9fP*jfOMWDAEUZbf8bW#1;d0Gn$;>GY3(O;s`U;5rCNHOxTJBYAEH}y^eP{J<>1cmBN$Ucaf2T`oE8{}$+ZtM~^nmZ$m#!TUX zPkm?vFQ^mGxx}qhCQO7my%)j}gR=GMk`H1X z&8@F4vu`W(6=Rg~x4{`kj5fX+$+7X3lYI*0R2TGQ2>SK;LkS!q_JT1(!muOFGv7nS zt0pecGgEtGnSDHngNblh3e<)3;?E6qpo}juWQ_lQN^6V(4VF6O8u5~eA*RVo!C}^P zL@IS4gR8V$tjKKf63b#R77W%~M=%_e25+$PQOyKHWe(ZcroOC8_fHOw4G5Fg^^yt$@&KnuYDP%ca4V{vMnm<8 zTFg4gZI^f6tCXP$y&GbPqP}>sNEz_`J$|36L@HR(I@f#EX`Wk9cv|2+IkORGyzYX% z#fwOZ^6afp6reDfO|+d8H>R=)lLRl{1Ld-z&>0A!Kpu$^P{*4~gA8lv`BVT4&gmFB z%--BF`BVX0hp>;JJcBsE8@Du|ZH#3ImALDHNe6F4E_5~tR0p8tY=BMdL1WHv!;2bI zyp*Jl$7?mSH;!gb5_V7?{%W42AX5~4G4@L)Q$0JHLLf+AteuI2%mgYoFNLw-R?GVV zdAUhx;7HRA#k2fr^a*CB*{79Ur$~Z*_CBrLuTv|!S2WG^)Ow%bH;RzyU_K1ww3f{> z=RK^V?-MN^EMaw}Y=99cw9g1iZ{>3oEEiG=FWRJ^}4Ut#6! z)D!6%*+U@L7`l?69xojG81_cqJ_ycf?i(Qk5jHx#tH;=K0~g^ejxx!Sl!*>`DGzTt z#4kRm7sILe! zj~5G+$q_{oAeW<;h3t$K(e+TSNH^8vs-7bOMb_EO}-R&0GZFj&B(!H*M z&ASAKfbM+{EO!fxqOUDrr~saR3u~1M-_^$llW6lx&ucjg_TOp*#xs+Z^jp^cwj^7x? z4)1>Wyj%q=!5QL`J!M^*d*shQ|Fn=;%SxYN$oZ@^{WMA=UPfv8-tsy&}0k6!ib)uloNN`7A9N%0C_Dpi(C4Q*Qu&c9in0UP4@Tm&J>n z_GW9Z+gfZMG&e!Xh++hS^0u+Gba8RvdQ{ns;?pH{^JGcY0RS692EhV7P+d(y2pTS? zV*q4^h&E|{IMap*r8eqz(%q<_jap>eaN-X}0rIv_sP;Sg(xi6EHwI0-1qETLwPxkT~CTGzY zA6SqllnnZ&}Vmda&?L9w987xdEf~@JP+b1#GNAn zIQL?D#TSVA;0Ty4IQN|`z6`KACWnk})>)(Hwc`rx@f7ylX({s<5= zLAUD$J*0-;@+HR6=_w3&L+tPul@E)?N1$n9V<3y`JfKQf7c&%aO=^hQ%JVDWJ`8QBm|V=BtQxsNctdWML$UZHyze`E{`6xzV2zD| zKZ&|x57?z=pM=5-Ikz^9Z0QW7s$J8u?AYclWLa{I&vChWXgZnt0}?09_&A;2tOYK} zuIzlL>^sDRY@8?*_(I8xE{KK=@Ajq%-P}ccQ7qTp)?!=q+@e;$I z{-4w*+O=@g`sd$+5M-TnpxdG=>AtkEst}LmbS7^#LxgDYK@V5O!O%~NDF(HQbgCZ}ff9j{D6>HT*O>Ew7Kh0+=zHyVI~Q|-1oKesxxQLGjo)nrXh zYNB$hiGPgntR*es{GZ#Ym%g5mzM0Irlq`aMk|~(MPb%}Oi5*y)TnWkAgZ6T{Ve)F?f*z+9?!hb(!%Zgzo$PG{r{e>eBJ+jiI3&dr``GX zVYLShA-xpsaNtTux_fTbI3}OWs=^WnAYi+sH%1&fR&AH${)gVo0ht@%Wr*xnn}kB1 zEx$?_e3E3HA?bqn7$_r+pSy1lWu?QAqIpZ>GM`2Bt_IitF$Nxqg$L4(&t3ove0`9D z-K#S!1Y>OR44Nqv%?&Y!REE8dJn#^(oVB}jW}Z9wXt(6)!QNII?ekqb5}qHRCjn5C z5APz1lQs^}Ljav9WfoUawslicK$=Y`UoaFBuk3@f}JYK ze?`5_NH5c2dj>l^k6*c3w|>eJ;VGV@!@u%3U3y=qKQhHf)ee&AuPPPESDM!KSD6Dc z_pCn}M^puWxb#EjMhUGKXp>J|{H8k4-{re;5(w%mR}?E^=p%lR4GKbk*X@In)Wvvt z>tX>`YIk{|Qc*^7DCMF)imM!Lq?E$YPt$Qt85OZ-x=2m@zjaZCwfe-5tK?B3GsFwA zf|L&R(m>tEI3Kcr`QB#|pu>caaJVTylwnrXNQ_51${T)?G@=2!GL|1;r69h{CF&N} z^;aVh1!ovbuhD08r7(U$_ns^pCC>udMHnf_e7=!(Z5gLc?vWQRCTXog9W}=Qjs2XR zU2d!ka}3C64O|zDZ5qplAiXZCSO|j+1>>qsQ0|%R5nl?)dZ~@rltDz6_aaN$)5VaA z8}jC)rngm;h>aVGzJDGJK%>hBEgV+N60RJR?ECj(#qL1?H|cDo^OzA-rkpjP_kJ&Q z7uAUYUjHf#!Kpa}HG>V+<+&UNifA&sgj7r##UJJ`)w{v~(p^Ec^jFI!1awgWGJ2~J z!CX8R&$_-rX$72MUkCdY-_ia2Kx7FAKaVO^mGdO~u7F4YJa&aKkcj1mp@cL4-#l_# z90En`T$@J<*#$0KEOwok$K+vk3z#k%7J4LEpRQ;bBN>fhf1dXFBue37Wl5QjdAyfX z7oTxR1GGd2^4hBGRgHbX^enaDFu5XHc|lH_d4(x!;RgSfWlHP-T47FWM#ykLP;lSMDBuYP7JtmFpVWA#}cJ$j3omLi-n#9C5whJ z&#ZcVbVIZz(2{V>*_4~0HMRIq4`naEx@Gh&N(lNf_ zyB3x9V;ctk4bd+H56I~HY+oiRFdS4uITv<+-R|`__jh-j9u!TzzdwSsX%ra zBBv@kacLWMIgIm_d@O<8X!$7;{2bGNqO?`b?8*bT@yfUhB=o=>3T zLaSA*U)q>_VgN0bQAh`$ifMF;E}~uxFCCEPLEEOf%0L0J(}FRP#>swn|FE;!>e{8= zY`^JmwGMi3ipjg}y?%3Z6Ea&{{qE*|r&TNi!td{We2BY?Qm7=qwRza-wtsHTD}>&Y zeSCk)PVTgNonP%FtCNVEm2mJYNXEVXZu4*Z9c4}9oS#9qY!!)6Q{+v~RS0M~0LeK= zi&OKr+ZbJXZiOXtE0geL^1}0()_PL9qK$i5sFZg*<|9ym)Y#;%ap<->T>#g5K^tm z1CXlht<^aYXRa~@Qnst=uq#|b#^N;I85u;N0r}I!d}svIVED{phFsom!p!-ly|=dy z|GcgXDP--!T4+ikq$*4CG*9%-19CPxGZBO&nB?|rDgDH!c=<{0$&{{4uICd-Xc{oc zRBzNpXL6M-aGR?jziUvpZeA8-f zq3Z&uY+VqhhkNMkzgnqOI6#jP0rN2l^?V>c6-(y;i;}{68sJN&6F*7td<3-WZVy`M zSfvTDf=F?6K(%qXA)|@Ede3KmjQfB7XSiNSpWDxWR|@{WPgcL~|G&iNLD})|0W2>b zJ1h8u%7e;-k3Rc-@bO2#5AJ;OiwKjQ@9yWmSmjfk`abvshrSO!-I?!$Pjck@;MNmg zkg76v&hBM%I(6k0Dl(YkV88Z5PO1ofE{r}mlmOz}OJv1O<$tACV@f?lnO~=X77cME zAA_Js1F1KeHpB`=<$TE~45NV*$jz)>_9mWqA@np6z0Cu|U5I1Q@pY1f3`W3*NQanj zxg_j!IRKJ^B9BggGKBC;b;d`iK80ydSlA9tVuh2XPm*}D+{c0yIu(`PrALh<8ocw< zx@hp7oQ44aexZoMXr^9zEGr61>W_o)9bQaJ#M`$N&QKmvNcXzG`aS?{d6!@SDzS}qXx!i!azmd08hIcd`g7Z zfq?70;V^CxKF7w{%~d0MhZ7d*eJvI@jKgm}5_pt#&U6Ih8V&i^^wwhEJHp6FwcOPn zU0I%I*?5wBo! z7bW?n+rYL5(w9UT6E!~o4frCpW`}{*uB$Zzr6i4}x*g-pZXQ4I6zePG9f`Z_mmVW~ zz`2Y=MO-ya)NCq@D^Jq0PE#y=r@H4~7(*i%x<9n~g@C5P@EVw*=OyoWLO3Ieb)@=0 z?Ku3WhCR*zXw1YGXzJ54W0PvKN6Wn!Pa<*;{v+vg)!5qqWv?#!u|Gnw z$ipKE+?VJUII}QJ>Oj48q!{J$cw17npsHu(N^vdn6>cek}}r(gw7PlnrKzp z3q^+Nn_ll=X~kUu6?a+e|NU+JHr(WU1&cd=c#2mf9$cMV;>J4om(P{2u@wII;EKps zt?2XfzgltsL?7J${T%ogn^IQ6C`>+e8@HYRK6&;mxBq*#vikM??=SMP?tkhHAcn5@ z+4uL}cQmOAZad`uiZ^-k6js_hMI1$ZpJFr{kJUv?fsz?@d*goS+_P!MfHrTH2XePi z&2jQsRcF08Z2vt-m*`*jY$Hpjd*9ULQuY-~R9*%cJ9&h!PT6qriao&V;l2=a@PgDTsRn1^tFyReC9$Fv4h}lVE_0OB{u#HM5R*oT{&Dx76C}?#`%iGmgE> z7kk|;^&rTlc)TGV4>eY|K{{-A{1J%rGucJ)IJSPTK?g|AzSJ))vaQ-COQo_Xobw3Q zy>sk+Fr-^mtV18stGU9+I9|*AhteP=U*MP3Hpcj`8d+H{Fvf-5XW`+$$MW*j5xKJ_ zema|Ka>_=GEW4*xK3*`zp@hWitS>e|*&W7OCjw?i<{*0GaWpul{m(D3Ip4-RtqrP! z1aMD~&$353?*2*A%dy`uTIyoL0ltc8kyB;c)Wf{;Qj$?`D&t`QC?8fvP-b^QDTcC# zZOmyJ8QWw3eE>Cn35MyJ`16iEoDb)Xk7CIfBZKC#e~P=7coUC@8z%WTDasNiADb9} zXu+HgZVe4V^0JDlRg)8Flmu)ayR#Gg+OV50;g8(>F_^@jNs?1?#SM0`(8$#VgnH;h zlM+5R64xV1(0Iua;v)=_L7=uQz0Rf$x6(RCBM0!9`I{ao*aY?N)E|~JTA`aI9k9rz zCYz)c=z3E+jq#GpaNy{l_}0MbI6CphebDJ(&$!j@9?-Mt|2S+Nw)zL1_I{_``?W6* z8d34Bi~r6x?t5Jp6_4rl5R@@TR{vjKy{wY>sWw3TYzJ|Y3KT+B*ko3d*-4hS2C`uQ4g1OgigN6Vbmzu`E;aH1#mLV@Na+dK85bXS z-tkU+{w5#wIa1ToEIIU3FBtdcl-IM>tbzuTB#Gk79^Yxo(X*+tw_xuJ+vHj7LpL#+ zMig5ysSwz0%qmy)jQ#JxrNdy@r8VPzY+((5QU)Na;e`R7D-9dJwz?lXV0+!2=+r@h z%P*lc#G{whdTzv62E6>AvuFR5ACCVNgJc>1{<+$?E&l83v(^0h@6+Y4`~NTUSzF8P z|2x6pOuPvO@BB+wzt?#pvk z4BiT0T7z*g1uzHXA+66SSMUV+mIzEfS6zg%?(}S0LvR;oh&zW2;#*$I?7-*D)A8c@ z%CdWwPR1%+71<%m<5K~^SNLcM1!J;0gJ;hnhSmqQ0}ar4v`wAFvtb##2{52-9`qI= zQD`ipjcRPH4o@MM4z}^`CLWHnrp1NW+nVAx@*yq zb{guW^$3=d-(e|nN3ocWZT(6W`rSMC+#sS-K{Th3KMbh={GC%GjiPbl$_NWo z7QW*fXccGZk3!P97InCd>byudFqo|^eyf+ltz_}SC*G9q6!jbfF4`#_3dM0*M^`=* z3*=!k$m~Tq;tqyVIl|jM{51Fyy=}S32}ap$n|Q<_96228IUD+pjk>}a(YA>|)Wn18 z+vI!0UVlb(g2om%Ht-D-U9w#Nb5ie}zX($qUv4Ic5HdJn@0L!+Uij{(e1LRa6FK66 zf!8CzrSUd-TotQx>&OT59yse_7288k&_}ho#fi9A*W4B!2_7B26X(ar;)vSCH&)_B zI+l4?gLD8SS?a1GsVI!M5CsKc!!P((CP@k_O3PG5MfvwS;|>a$X>vyg6MzF0U&tFE zAR6Lm8sjAz6+2Th2*;R}#WNmJp*bDcEiMMKp#*xWx*}`TK-5q_)bC^2XD3i+Zn2sS z3Z^Qm4~OJyfUxc>$~R7;A-?uh)zLYt=A@uQ*~LV8m9RHKJ^-C^EzsUX?8Rrv3?>36 zWu0dyTTRR&T5!Dviw*>A^?=E49|B)|?w@v}w%y3W6)C^d&x<<$PIye9wWd{F;?1F9CE zxw`Mt&(Owg`v3B3-v9IIlOMkNe}0iq{{4qlApRnq0shTTU&SCsKv#5QHzHksU_1ul zio?V=Zfla+553;e$&sB2wX(0y3(+@JE5022|6@H%0mMhUmI92s`j#@%B>eC0UCNa{ zg@38_#>9s@n97j;pTxryV0{V~Qvh>|k10JO0CawbfIuC82R!JqtBJnbXD7RU2zw9T zeLN?7I-Ws(GB;o9D0B14mj`vzVhOyYkfpwhQqXwcIII zw{uk9&M{I6b**8PX;DhOP?FbAv!wK~t%(NAd)aIk*xjdysEJSPc*TbZD%Pb0B}rW_0mD8whcyWVtv^rD$cR{pzx`Qq_LT{OeX4fS|) zGE4BT2);<56iVl07T{Gu?yn3}|BBOUDp9&UWhWlgIgbq45@G+nKY&&hxJ=~6uT0EESYn|xeEaMMn#`KDH!@Oj zUnjV`^uvAar&bw4&+;1-;orZ12U=miO9!!IFM2RFkkKSpI`kmD#frTRK)z=n z_Od&hl5}*chkKiETAP1wZS^;s7^5}cE6M5{CC5vC5d@$*EP{QCKK{hHp7G3-=8CG)G3B)PXG>!(yzwm*)*Mu5)a>E&c zs?9gNv|eht9HKU&A6LL|F#JLSbO7~K`IG`Ikdf4A-yM;;-L{65|1jD@tLV)c0w;Lu_%U8&O_4p>f#aGTGqwO{+h~sD8n?K z(C#4<-$S1?W_&VzeM3CNnpZmQTHvEn3-yJ}FXWF|gpnK0yf`bLrCn3`s!`rg2Huq4 zY}e&E?V1v?{M)xi1!bZ%D=2fNtZ;N}goz|_+weJ!;V&(jQJyJ0RXCe)X%!2yE|_xG z+4w_tV!LKSlW?P3!6v;q0^UPR#sM@*PQxdv+cHb<*Q&szGz3F`L@$);+S1{M4i<30 zMrU=g@S@@49dL~8eAwGw{IRxRw5PiTEGgUD^g>+93CMyjxX=&}zpcs!0QSq+#|J`^ zj60aZcwGzq30^?s8?ql~u;sP#Ii32xxRDLbx=QMk*_?_xX1$tSQ4IywT#=|rdp#m# zVq&Tr5}yo_QkdU=pJj&VPZ5k1$p0sn{`+n6_~FuRB_1ppE>JWefsap)ly}tVA%5_U zCxw+_$M3$w&>~})IWHBgZ`XCwRi2{Pg&6cS#0gywA324i1>iM+DfsXKw#5{e08c2X zSC57bp&zM#G_1oeI)%U0VtDZaF^fg!<2=8t?6Q?_5bWBb^&@6bn=^0ZOZ!O#;+WZnd@Tw(3-e&Vm ziV=flO5_OfkS(@fGLG3qrQf&?EM#fqffw-$f6HrGAWF<5N$v1S*3y!0iy52&C~y%K3f6G zF*VLv{ULw<=jqB<`~MgDl-U2Diw>+bZ@jclH-J>kLX3+FBnZZMf}GIv%@_=sEW_rx zfIzGK`xsm6MrxR0$;l9boO`u(#|K-J^8CAB^hm4qSzU?khK$p@BAUg8Z6B$ zO_uIa?hd{$b5lOukF^TCmr)Ddu0McS} zgB8wKOM^7tmsj!?%*3VjWG_ma6qMnX#`5)atMbjRTdMyK>YMxfMdCKcrxA?n*#teS zZz=n?U%GEdrM$gdJDkqu^te#$uBb}-zS(VV9>mdka62NhMyK1nxkBvYTvhk;0J9_t zfALRijQ zjaQH?USaZ!C2xZOC_ko$9sfMQZw;{`9@31SvcD%Eb6MKQiyp|E)DK~YW#Yb}&ZE?k z5uUvP1lWjsV|jaA?Cie=txsMhKlM}o=GfW|wTZcV9)|E&FR!U5Dzht}GL$i{W`EoI zyZ8YUJSE1-BTaj`%1f=Tg0zoo4OaS5TtqX_Fi&7JF~KK6(RdbNypn8+t7z%&0h-Bh z(bSc2NgI-JADGpZh~n_z=dsw9HGu*d+7xNoAH0+AXP#rfFJUE_;0u8<`MP@IZ@=_h z1?43@xhY=sHt6MVweY1#HR-)2S8?j8#8FjWQhk1zVbfN~Nd1|{S`Ju-sP66!L$363 z-DyVo06;`&CG-S(=fW~dT^2G_Gy8aP2hP>FoAqUvQ)WHbP|rMyR?RCg&^?U8#c(zr z+qGETx02|fCK08w)hM{6zn?_W*!R#+I!T?LNNeII5Y|rBCM0uyMns!*-NRRZYi;)W zt$((=7yu8?qvRvKp%XALWe*HyFW8r2;8Z1edK`cqWtmB;x%Lc$27ZipDGEEpwZf#NV4N%(h%P7O z5FTc_6e)or^r zzY{;;5Pu6ykoE885uT%AIIv}@a{KaRl;c8u{0Q3==7p={d*jJrJmw+SSL2hpq`%6V zEnUZKfP05KJN@=vx7Xa;RD)%eaA+p16@fA|ImcpI#Vp62v4Q1^=jl%`+^bA_(@TLZ zA?g?5mmEePAsC1d;;^$rECXAHTlhtCtJ%KmV}#(uAvxAgk{Ku-wOg2Y4mMKnmGUDX zSgvR(3-Ce}UtF^K{=KL!d<2Wn=WrSt)CxWsi{~IJ_U`LA(ffaI?e%-V9w?5tD>%F{ zbG-yNRF2kL-K<{!Rjb))b!zh`23CE+?7qQNM)*l_g5=tsQ?GQUWZCo_UDig)_#u=ojKJ? zIWAv_UysT)_g8Qq@=+1Fj}X_pQh&<>y+s5vF^`Ugmr;oLL)9pMnv+_(>ZydK<{`|N56*N?V>KFOcolYDSrK zK^fJ?-4Z-(C0u1%dWsqat;+!C-dTyNi_*-kgZ+b=-KOGZGQ^l|WusPFw3F>9iY1l0 zrtIo3GU@P>;DBPijSXe*Vtvslz2PVwPytkiTJifuVmH;}G0l~qX93q{CBbvc-X!J> znj5;btnJ(i)|>?-e~k9j*Nj!w{D6YC!6=SW=64Gtr^v*0$+(JL3+)?al|E~-zb;3$ zp@hNow;&@4U*4i>%YtMYTj(JSJWFC1=#UMdS1;)rlGV_)x;3xtgYvdC9+jazRM|b{ z1yQmRt7dxI9;(LyR%KWX73)clFjwqBC)x0duw*WV2EfqeBpI#-Q{k9eURx@iI}wLQ%><`yOeKVouE zW>nAPs*RxrxZzk3xZ58ZVYkHV8eYm_N+pIbe4~ zc|O3IiIPibR+r{`fJtaedzv~!z+Olc&2hdK(Mfg`@>w|M7k-~n%g{0javPh><&5TV zn5g&3^*Ksp9EhA7nngnseKyVpU>s~@_9O=-3!88RISDlsmqFrnjktx69F$Wj`c$z0 zA1&@^jS}a9r};rG`5cx4&)8jg5k_&yizPt8L`T`Z2^nQbx3PiLGHeQMG14<)S7mQUs!AUb z!{#pS?E!{G%@0(fI|MMU34r_^o{X+*HabfN14xeNVFd%GodSvVv2cg;Y*o1XC!yI4C2&LCOLb zV#IoHaL)m(Yvo?e#sk-Nvr}VRMrQ;u=x#e=Gnj%qHB-hL4x8g~&Lo_*RoKjNtL3`# zm!swU!T%`g{Z)iB7SCVM2PCD4GnpdIYiJPOpwNUEEHvQoVUh607+C3&)&+@gUQ4Ca zn>37B>%tMWe8z(>eIwgl+fuR2@c- zs@r<3Eh-<c=3?D zU4b4a0lrDf4zB>=JcwqAiYPUAuano>%XFAKkx475HykS8exR%`fg94(R&eC4Im?n% zFF7dJ?ax*G|3e9Fo73Mj9GGo`&UxuKbm2skB`Nz&4R*GMqa%IshhRx+1BKLB)ya{b4pF;<|d7pW*kdlKm@WQ8>8ybx}9m&rf1^alv$jrIp|6(IKW7fPq58f$q?tCnd6Z!Xx zWZogX_OViDYaek|&j!)Ec`#xi&|(vWC^exDN=@YQ5Cum(b_%y&M8WQGC;YR}$V6j2 zo~qV`U+l;odZJDb@$L+o6Yyj~KKAFBXG340GZaLcpM8*M(LvHcZik{;% z_V?pTNuqLwlU#n27%ZI2N__O)D#VMw-5s$)yRtPLM+010yJTf$(V3kM-H#&HN&7sb zPAR{UN3b^YOB0twx`kmt{GoBL4JQ_vo?l(2e&XcsE4qih?f#Foy6Uyjol^f8n<0r6 z=3b)XVXupvm$9~%n5v_N1w{(pX>m-W(y7+ zGSa4C3RvJiXucnY@51OJ#7nZXFquv9qMbf;)!++KnR}YR<^m~tD;gA7ywhzqN_hZ# zWEk1zFC};Gm3)^0&24kmmE!2-G5XfHv}nwXx?w528;+lzvF9Ux0#f^3u>#Q_jg0dL zz_9s!f&pMF<3|pFdC$}ufy{FGA2R~-Y`&cSpKF%pR>4~|M0LP%pQXoG!tcK#D;;yr zid9$>OZ`-7w>AH4N;FE2Kvd)3a-K`BtXWq+?D`8H#LK-9>rjXpC!j}v{;(@QK)0$* z<$gfXRNkc3$6)gL5hHl*DgA@#7^}{QRvtlcH+7QF9R7WXN<@E61&+7&DU>uz~6g0x! zZttO^Zh2?QJAqeWWItWpguI{AtDDiEHwAB2uz%++Gs_c?XG*QE9EIU11EXduteTl+{lL+T4m-6AK>Uof)a zZOlQ^{LG)__#vJ4KGWNvK{?;5&zvcXA5%ma<1tT4Om7noA3D($E+vzF4q6sEcb(=% z1+0WNlBOdMHY{^jUp4iwD}QxZtFNCselY&qm_w$-C!eW}TjD=I`QiC;KK}EwA6CA` zfBO=j2dL-%%Mbf}AEG}dN6K-7q-Ui(9CQa{N+LFmm~Rx4`9MeaQm3jXm*SKns|@QL z8=9ii;E_U|T)o0e3<;LtK`I4b0|Wq$8G+4uLlTyESg@l(K&QCFXfT`T`wSTLEuk23 z0HNBIb*eRLXy{Xn98v;Q7HJyM4BuPiFt7BKV-OC;vmxW53S=S52!MPK6Tr?eL>=N< zmyxXDuTPz$bu>Av1C%T-TOHl3$11d)jH;K=WA)KA@U_0v)sH$-jO{lq(cRzf{nG5T zM7t{vI{QDjw^~~&yjwjV)$La+iW@*^{qvyH>Qbn;_U^$>yR}sp?Y+&N!>#t-Yw-$d z?(O%)PJ6fA1F*e)YSIi5w+%pcTb<1}kkEY9-f8!Kt&8n;Zx5kv1K6fGXm)z-&BL8$ zM;sh>4)(h(Kw%32@3r@~JJ3#Rx3$-ED;4!7U$qYlmj3SEp zE%>Fzk(UC;%-CFT>>}^}i!~+XA`RW%ZT0|0^-&^@9wo;D{w^-UzYX~BZCK^zyflsb z@`1?)ceySa7zL5@hyDl}57NLJ2meyK*h?-5z%k@F$!5^!oQFv|1b(d9rCbb!1kF%+ zc4}}M(oE9VdAbrdjqi1T$G2#`Pk7Fcw>0+Iq=Wy|X6$gXE)^dQR(X;@? zO`y#@DESk6V)ZEF0N3|GEgMGS&%{S7NqYIGti<;p`jVU~w?<_rev)7`!gUb_V?~NT zb>mrf))7===~rG&hjdxq1z~jGZ4Eg=Zb&Olz|M=la z{{6q_t6%lMFY;NkL-ESXZZZz9)|Ik%KZMilZr5a}RMK$CC~biTzVAMpcbeFji4vwFZ6i zESOS=TD)oo*eYBSBRnwl6XX8gDe^Fa$_F3*kjTT_vn;r+xc{SrU!xEIwb6$q(UYkc zzjKp6#ya9I2~JRQ#$%tOdCFJpIFj(I*=Xd)cG=h;VOckxg|rOJ!T`NWeHRaZL7laN z`C9GL&0T)F_-YH6xCnaIoCbB#I*-QZ_!98^y3>Fkc-LV^`*X`e|3G^IMffto#}d7e zo^uD*8Q)SVrNB&G*g+2H>|H%*sMi^_3?D)1W4^!mGKGy|g3lAuYkUN(>n9vHwrl+2 zRqmzPaq&`hoa+8D^W)1FJ>W;R|6%>cHBs4N5uwCSk2=xCF^8BTEyu@!abPeLeBp3B zn}m;R4FmgiX|Kb^Kp~&bQZ9vh8rDKJ&FQk&(k=XO#8N%cZ9*>w8J+qgylf42I=^Su85!&m$7m-#F$iSEJHKNqFWv)CTO z5=_1<4XG28HVMT_48c1)KkusL%C-kDEn$@dBU*@Awmw(!7U%fX2OR=a1$|7GxCqdE z^=~{rpOE@1wO%)U&oSB6*2CB=xRuI-U=-p6N3i|oR{oDi7G| zb3+LLBCU!qw71ZPkvJHj>SLv_S=~%ZAj$Wvdc{KLK&2?&s8mp-fW(F^1ojt|lqIAC zz6wS0&sj*xS629dJ9`)qE#_=#!m4)B*MERf_>UK+ZZb6O8VdQmG$hm8i8ctPJKMHm{Y7)dIO4DlV|S*NBm zY}7&OTyW&+CH;XYBaoQTkO=18q0H6!Wb!&7~{jb_EH!H@YomFbSnFHj-bdvP#H5XV$9j}Cry;-7+wfxSWI zz^9z)MF8%hJszsp0IO^S;IG2mo8iExINR5`5KN?DKN$64r-Ed0saXfsc>IA@N%)G? zC+uX`2nu2}2S#iGi3?C%y!oyDe90M;P};y(y@w@E64mNjk&;Fqxqh8$Ytn+41AW1q ze1P>hy6|K44lSo(w1fG#5YbH6#&Gy;@FDGyP^dXvAS0{%EoqX^g_;qeC@HP4{$a73 zdp4U+Z|vD2=Kr-lBNT7$nT$e_aXNErlD!C}eS=29d)-y^gOMdM?9`lY;u&Beogpa- zXx@tHsKTisQ_99mv8DMZE|y6OlG4(7qQ%ZJIHWu z?4WGw)(#rA7VS#)@}Ie$claI{Fzy&OZf8F8P6`tb;{vHIek#FI)n`th6U)`brW=>5n{3> zFsYuT3IeOsk|M8PI&{o#tWen6_jqew7buam#w5MQCuD3@2b3*fOWc^1}~b_kUmFQ@*ilPNyI>)a#Gzm4eKnp1KnJ zIvDQr^3OQV-OqK{&<#2KUVQdY5^omGy&ZAlT{n00D{t=afxvim*xT=PH3aO;I7nz? zTHYyT+%{gjmnoak_2Ke!x> z$r~yLMJ9nQ0!A$IRKR{iRPi1U2C1Bac!Xu{5=ikoA3BJEH}xo9 z1$q;~o-lz85cdj_4-*=m73mR2XYxLkX^5W7p==~P7$83Z9z^Yg>DV)s zh_SrMwk}Pp4)F0H4o--IskE-kqUbp@&PEMCLw^KxyyCoB$hn61u^!DMXaq+?Vr2j& z4-xYC!bgAdGp`}uI#9JH9ckv!F5_a&=Jz3=c1QcSrDMk6IYK(MJCIUUzKNJgG(Ine zU%s7!RIf-krqEBHpF*Y$hi0Q@h4li}yyPd*E04pXiy8(U{T1IL91?UjN!Rn7U2;2G zj>iQDucQ}8nBijS9g8a&@@Z~Za+rm|xC|%l`!CsmJ69Z63`xnGFthQbbSVAQx{~ry z6JEwh#m!o+gpEtHCB;yUaH?92GG>}*|2>XqKI;ed|83YJ?nXm@?4~!Wz#sa#S^r;o zy1JbA|9`sj{OMQy|4V$NCi@uGIP|_|OYs7JUC=KECx76zfG09f{>rki=*%B=PWj5h zg6l59|LJ5n=?_oLqU{hwJjxN`xU9yKonclX2aZ>>V2t~Vk|t!qg_e(7Ya0``~l#Q_40-ypo>SyXr(3_ zwXNRa-hnPJ;`v*8x$B`-Q=dLhz4WXh7Dg_9S*S~i8jm3jt!OTOm&r0Voyw6vWo8o1 zV$^gvkAKOmVAx@gC;bH2Kl)N7eA^(mvOl~6LtrSddBH^pcuJ6TzvyUR>KR%q4{a3&C zyRBwt^Ua-!+Q7SG>$GCIpNG2t$g}7_K7_z>K08#TA;Q_@#E*}TZ8-!o6Po|+;gw0D z!-K76uhnnA-rMiAwwO@Y&ciE$WW0KLU0eI@UMv^S!W)TVkO2^zBW7Xa=Eo^6gNcK# zP7Quk40&TCcP7MF(V2BHNY{qLARM`&!$UnZVAar%>dKT-2GY3 zP6^pKek{%lq->$EdwVI#Y`APgH2Z1F44h5mO{_u+aUgLt@ zX>r~5o`6}1UvcJ%#;ujr^ehqH^K}>|)IEuDH*rT;Z5L&p)@?|$LA@dvnz{tjfcv(x zrlu0IOa@Uu1e_(2kE9cj!Bg=Z{sQgCmZC?;<=l1s#$!YcoNP?lH8?<z!@~adpK$24oxDD}p?YJmomFHn$#M4W})W^eP&0_~CMO+Bh#^uW0*INnOP zwiqH`tocZgd;5z9k|KbX6=MBJ&NvSpyGFU4hrf16~t02&b2ezTVq$?fyCKH2%N zQ|>RCas{Bvcbq-TcVhVE|7bQXSY_~X-XMyH^8P|yC^vg+|DT;MnU-w^{xwk8U;V-J zzfLrRY2{AOKK<1%`rPLK_vG1XKK|Da&%U1jeUZ2`=Nh9!P| zbuq;dp1wE_{0qKA#?F8UilaDC_)sq?1d&jY!|-@ZVEBD`mp^kcmOXmA8BL~92$Jb7 z&H{STzJl=_iurHSbeeetzU}z1^GMRQIi)4Ex7y3`?Jflq+KbTQ9$olZ;Ox7DK-U@U z5&$7g?hJyKI6r=S7?0)Gb_i2`1e91)xAR#FP?QA@*dbGN$%^f2N4c9i;(>S=V}jhG zszN0ViiZr0|)!T#i21mnY6dc`8d! z*XW|K+f+R&ef~m|h=}XRI|2{Uyn<5-)@tfOjt+8(mc4}qUJjO1Tijqk8HC$gl~clz ze}Ss}WlrN7Gz|b3B_s|Yy^W4JhMD=He6KB6lqZ$-xI;0Z>_a-V@yUU2U3Ip+d(=f0 zuXXNp6Auf|Gh}hi=hkux3SR=uAEqN`0sXa*12ac0iYxahsV>x2*Xzb;mZ+A+q=$BF z-}0jZdJa25cPxWSM+-9a*}`$*cH%>i#TQdPq4SFA7(}eI@aIBZ%z?6I!Ju=7AnX%Y zI9dN}(ENJSEm7p>N>wKZ?H)1XjXvDS^uT{PAw)W{opj@jGsVuFM!eyWAnGa2amXl5 z$}MOV57usyaf=%O$B}hqExTDMhRx1tVbCa(fMX%9 zo*JYnPfs}EyY1z?Y}3|SW&T(A@_on$_5Yy{6Zq0i&OTEcx9R^stgPhq|K}^KU-kbl z@pm^Nm42JP+b1L01{1_p~Vkz%K@!>>qEf9h5$ z7$IoUpG9ICOnnT3&asTcQE)nAzvPPcxa)3q+6TSU>9+Uxu)u0#@u71XAl=F9-4~0C z6h(A#kzXjOnH78O-B!2P+&v(yaS2$QiSHl%`e^cK_~^}}-A7%(nrJtUKz~BjgacD$ zAUdtB^@X>~Po5ktuRU3r{Qvf@wyA9-34iBT%xbBNyfF%py-gJa5(6gZDq>jUY$}yY zse~FIgzdHRtJdDUyTyEbgw4SUx< zIXQi+0=IUbK6L}ThdLE~T}!i3$d=lAD2WGG8C5T8hqJhO-{~Hoepvt7J-iX?KY6%Q zibODoUzSMbee?cptJk%u#QtBlbxOtoUY3*!d-I+_PU{Kza#A?WW1h?xKwmH<(rgf= zf&u6zqdk3Jhn?GuaT5x7)*op&{5Zd$i9;%W|Fr9(=+f}|0{{haB};59h!aXiinuSR z`(8Z3s$&P-`zh5T&lcXIRyoC2+{p*h^$@`ZdZp6lDCN{-~3SlE2 z;k}lTExzC^WB9K1ubRwH)L=}?_zP*})|$$HO%vlafY)L#AdcvOX3RCo?IBFY(I~En zlfMDVcF!AMQ`fSy{#7-E8?Q?LK=mTxND7+Mf>lFAr%^S**^0j#?l(60%La89fd3j!ra)^Xf;W6kO>zLrvdDm5<4>2Rbd#- zFEfTJ$>)<0ow2LmntX3A0Y(r`^r##K5|yBE&c`V zVuq#Dmm!xzjvy7CqH%~SYRg-$X6xHEaUPj*$Q2VtxAm_?Ov_(4Y>Z4CO?WbYjCfC2 z1f^Q!FMF$1!t7;1tvH`jaL!xv<*ajBl>pIR8Vui_|D74&UMCUTd%C$C$Z>lkW(+2Xj&SFXby|zpZ zq$(SwSofVCN_Fg}foP~{6aWp$Wvtq;DvX|~qTVGrBJgTTc|K_v${hzIe`~TnNDc%3yCdo6R*Ks;$j{+@uBR&x5K9%DOT#?Hcz~QB|Z;IG) z)M2*Uaeg3ycNONaI5$;rS21I+zr@Y2#0@r)L2a)WfPZ{Rxohw_F_VgQcN> z8{Kzq@dO{^Yq{H~>$W}FfRnNyKBA~?iYt_aW*FOE%BdN-V>W>=rIL4hU)=_?g@bvE>yYzM1-IYm|uLSJ* zh6p)Q?@m(p;Qt)>5wm1a=#B4zMRITtH@J{H!eSYiNrsfzj`}ZNys+E{_M2v_q$>_# zt;g>EySzWHLtp?GfV8`b&Wf@l|CO^gpG?k98<3CDrycv#4QGAYk^joX%_l?LX*Tky zE8fnMZ>6$0TDDVqE5(xXs@0n+(iiO9B(p=r>QX0P>{QOE92%wx>c=^18h>2{`@V zK03f!lI@nC4}{~T;KYP3q7)JP(P;1Y?Z(Ss($!yZllBW#b!sIp?E z%JiaPGN-CMaNamS^>dKqgLxX|{?;}zz*Pn?#Q*fX<>C1@592>e{1f9NeDIA?0PdLo zKihm-_5VUYAMAg=%i{ugsr~q%+wQv_UuwQyT&FIWy7!kPTJCe@y5w`^b*Hc68Fc6B z>|d`4G)7mRWfI1V87V_N&v`HMoY9qbOK*Wak1go;L_$Q)Zi)0NrkO}GsKhu*qR}KP zFgR91PLj!|%BMc#fk~4hYk2fJ8RQv!s|0O~Z0wFCI5KEn9S!ECtZlFqc&-dG_sthb zT+I(er_y|_1&@-8A@ig}N5c=$vVrQp1i58M?bN6@Mdv6frda{-EqZCecG-TZ9181G zWwWnuWq%}HE)qM>CHT?td@^`m&n~-wK=$`VXz*Tk??qn`5}`$gxrI>2AZ!@}qfjHX z)syjhARMw7;-O=Vl1$i7Z(5bCzl6RfZ#U`7=jJAVzzyl2G*Pt15UqT~E-}0FG_{5h z&^?;Q81fZM2rE~06o3CZ@~fn+sPCq&i1^FT_Ur9He|;6t7w!ogznA2 zO08%SPODHfh_))M4<>aM{St&!xt3rDVKkwPvV_N0)+uLkq+-^pt0LrvYvQEyao7PO zE(%B~tM${UbD>egsc&9F)~U9jpI?&@ev8t1>}J=*Az&E~Z*?{%%CfvHiqoB0DFCX~ z8EJ$nl&dPQ1p~#)6KUQ#FwH4APV!=QnyaORKg9HT7@U`d>sEc;n#rZS$2Nc~ndGCo zGH)v6kii%`3KPwtxx7i}&PVBLCADJz$TH^8yG3_xte{gGNHk@D1>ai0J1HztgOzEH z6$;wDGd;dTC(5IxtnJm*5WZWZ;lSWf;yOi|O~7_Z$0__qaEjH|!8NK!UdZqVylO+# zGh=>65lQuZ^(d#LfFYQyEC zGg)LGiKyXZg*MBM7lv#?b&CEy`&h|Dg){#U7l{QI!Q^Z)Q*|0kNj$750i`M)QRyUu@q z*{+`dK7IDf!};%bd6wFIVtb+I?&e^_1qX!*wm$eQZ9L_hMb_4{dTp%NgzpLJz!8?m zvGFGxu-jW4{N}N}w~I9TG&cvt(GU1)#5DhncFd;tR~Dw<*QDi`t&P6JRj0w=d7Wx%57 zRh{9g?W?P7s!qJAP@j`D6&Ep%gfHi5Q&3Ff9JkedM`V2+>jXUPz+8cP#aM>JHg{rE_n z#W^hZ1s3rehvU5vyw8&;>5E6SaL%5Sl+8Kv~ zEM)~}^IWA*$^@CF8VQfc19qtCq@6-nF=Qfnfunxo^UOLnGm%t!>CuF#7&mC_dBEXz z%($WYPClD8+dalLmdPJ+h%-IusZ%`PFn2TqT4B;Fa;WZ)TGkR+1fJfdo#gyzL(S`9w;n^DUP7b=E))xkKd zvjXOKn2N|(eU^1me>>= zR8m}qrd`zIUz#wS#3^}TAGlN$DcC*vUqQbb+=ntn8R`mUucU(}7ih3@&(*+bEgFAV z5vEa1nXxhz$K^=WJ5m&$qm}eMh`%JTBegM&Vj@&JsBl677pUAi5XpY;P1~rauA`7O zhPZJwUJ8~~0f_3ZPN)hsTD4bGDkZ=fHBY#l|AT4d`rT?C3u?NBtYzQmmso0wC@%aj zCjX|mb~B@LKjw@^aW)R8Su&aVQ98YX@^1!zErnjO!@+CbUS@am6s0NfQGp10!wBP_ z*VG5v!Xy3iG?6sAD5!s)F*d8$Gi6@@+Zeo2LNMydH@O}xJjt<*Z|Kc%SRRh4Pa8+6 zG;SY_ifAEWlNjZzREM4^b+{-(DUM(|8F}z-wk74beY+eBX&bdysuv?ew36`CC%^I@ zvfTa&k#upcl!{+BeidiL!2JtKd7=cSaN7+9tDF=G4K+Tu+36OCG#(aR56{E%@O;1L MPtH>%9RSh+09sxgZ~y=R literal 0 HcmV?d00001 diff --git a/tests/test_headers_filter.py b/tests/test_headers_filter.py index 72b2d888..dbd533ec 100644 --- a/tests/test_headers_filter.py +++ b/tests/test_headers_filter.py @@ -21,349 +21,267 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import shutil +import tarfile +import tempfile import unittest +from pathlib import Path from scanoss.header_filter import HeaderFilter +TEST_FILES_TAR = Path(__file__).parent / 'data' / 'test_src_files.tar.gz' + class TestHeaderFilter(unittest.TestCase): """ Test suite for HeaderFilter class functionality """ - def setUp(self): - """Set up test fixtures""" - self.line_filter = HeaderFilter(debug=False, quiet=True) - - def test_python_basic_filtering(self): - """Test basic Python file filtering with license and imports""" - test_content = b"""# Copyright 2024 -# Licensed under MIT -# All rights reserved - -import os -import sys -from pathlib import Path - -def main(): - print('Hello World') - return 0 - -if __name__ == '__main__': - main() -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.py', test_string) - - msg = "Should skip 8 lines (3 license + 1 blank + 3 imports + 1 blank)" - self.assertEqual(line_offset, 8, msg) - - def test_javascript_multiline_comment(self): - """Test JavaScript file with multiline license comment""" - test_content = b"""/* - * Copyright 2024 - * Licensed under MIT - */ - -import React from 'react'; -import { Component } from 'react'; - -class App extends Component { - render() { - return

Hello
; - } -} - -export default App; -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.js', test_string) - - self.assertEqual(line_offset, 8, "Should skip multiline comment, blank line and imports") - - def test_go_import_block(self): - """Test Go file with import block""" - test_content = b"""// Copyright 2024 -// Licensed under MIT - -package main - -import ( - "fmt" - "os" - _ "github.com/lib/pq" -) - -func main() { - fmt.Println("Hello") -} -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.go', test_string) - - self.assertEqual(line_offset, 11, "Should skip license, package, import block and blank line") - - def test_cpp_include_and_header_guards(self): - """Test C++ file with includes and header guards""" - test_content = b"""/* - * Copyright (c) 2024 - * Licensed under MIT License - */ - -#ifndef MY_HEADER_H -#define MY_HEADER_H - -#include -#include - -class MyClass { -public: - void doSomething(); -}; - -#endif -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.cpp', test_string) - - self.assertGreater(line_offset, 0, "Should skip some header lines") - - def test_java_package_and_imports(self): - """Test Java file with package and imports""" - test_content = b"""/** - * Copyright 2024 - * Licensed under Apache License 2.0 - */ - -package com.example.myapp; - -import java.util.List; -import java.util.ArrayList; -import javax.annotation.Nullable; - -public class MyClass { - private List items; - - public MyClass() { - items = new ArrayList<>(); - } -} -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.java', test_string) - - self.assertGreater(line_offset, 0, "Should skip license, package and imports") - - def test_typescript_with_type_imports(self): - """Test TypeScript file with type imports""" - test_content = b"""// Copyright 2024 -// MIT License - -import type { User } from './types'; -import { Component } from 'react'; -import React from 'react'; - -interface Props { - user: User; -} - -class UserComponent extends Component { - render() { - return
{this.props.user.name}
; - } -} -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.ts', test_string) - - self.assertGreater(line_offset, 0, "Should skip license and imports") - - def test_rust_use_statements(self): - """Test Rust file with use statements""" - test_content = b"""// Copyright 2024 -// Licensed under MIT - -use std::io; -use std::fs::File; -extern crate serde; + @classmethod + def setUpClass(cls): + """Extract test data files from tar archive.""" + cls._temp_dir = tempfile.mkdtemp() + with tarfile.open(TEST_FILES_TAR, 'r:gz') as tf: + tf.extractall(cls._temp_dir) -fn main() { - println!("Hello, world!"); -} + @classmethod + def tearDownClass(cls): + """Clean up extracted test data.""" + shutil.rmtree(cls._temp_dir) -fn another_function() { - // Implementation -} -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.rs', test_string) - - self.assertGreater(line_offset, 0, "Should skip license and use statements") - - def test_python_with_shebang(self): - """Test Python file with shebang""" - test_content = b"""#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2024 - -import sys - -def main(): - pass -""" - test_string = test_content.decode('utf-8', 'ignore') - line_offset = self.line_filter.filter('test.py', test_string) - - self.assertGreater(line_offset, 0, "Should skip shebang, encoding, license and imports") + def setUp(self): + """Set up test fixtures""" + self.header_filter = HeaderFilter(debug=False, quiet=True) + + def _read_test_file(self, filename: str) -> str: + """Read an extracted test file.""" + return (Path(self._temp_dir) / 'src' / filename).read_text(encoding='utf-8') + + # ------------------------------------------------------------------- + # File-based tests (mirrors scanoss.java TestHeaderFilter) + # ------------------------------------------------------------------- + + def test_java_file(self): + """Test Java file with Apache license + imports""" + contents = self._read_test_file('TokenVerifier.java') + offset = self.header_filter.filter('TokenVerifier.java', contents) + self.assertEqual(offset, 46, 'TokenVerifier.java offset should be 46') + + def test_python_file(self): + """Test Python file with MIT license docstring + imports""" + contents = self._read_test_file('results.py') + offset = self.header_filter.filter('results.py', contents) + self.assertEqual(offset, 31, 'results.py offset should be 31') + + def test_c_file(self): + """Test C file with SPDX + license block + includes""" + contents = self._read_test_file('crc32c.c') + offset = self.header_filter.filter('crc32c.c', contents) + self.assertEqual(offset, 50, 'crc32c.c offset should be 50') + + def test_typescript_file(self): + """Test TypeScript file with imports (no license header)""" + contents = self._read_test_file('FileModel.ts') + offset = self.header_filter.filter('FileModel.ts', contents) + self.assertEqual(offset, 7, 'FileModel.ts offset should be 7') + + def test_go_file(self): + """Test Go file with license + import block""" + contents = self._read_test_file('handler.go') + offset = self.header_filter.filter('handler.go', contents) + self.assertEqual(offset, 18, 'handler.go offset should be 18') + + def test_rust_file(self): + """Test Rust file with license + use statements""" + contents = self._read_test_file('config.rs') + offset = self.header_filter.filter('config.rs', contents) + self.assertEqual(offset, 21, 'config.rs offset should be 21') + + def test_kotlin_file(self): + """Test Kotlin file with Apache license + imports""" + contents = self._read_test_file('HttpClient.kt') + offset = self.header_filter.filter('HttpClient.kt', contents) + self.assertEqual(offset, 26, 'HttpClient.kt offset should be 26') + + def test_scala_file(self): + """Test Scala file with ASF license + imports""" + contents = self._read_test_file('DataFrame.scala') + offset = self.header_filter.filter('DataFrame.scala', contents) + self.assertEqual(offset, 27, 'DataFrame.scala offset should be 27') + + def test_cpp_file(self): + """Test C++ header with license + guards + includes""" + contents = self._read_test_file('StringUtils.hpp') + offset = self.header_filter.filter('StringUtils.hpp', contents) + self.assertEqual(offset, 16, 'StringUtils.hpp offset should be 16') + + def test_csharp_file(self): + """Test C# file with MIT license + usings""" + contents = self._read_test_file('ServiceProvider.cs') + offset = self.header_filter.filter('ServiceProvider.cs', contents) + self.assertEqual(offset, 12, 'ServiceProvider.cs offset should be 12') + + def test_php_file(self): + """Test PHP file — Date: Wed, 11 Mar 2026 10:07:23 -0300 Subject: [PATCH 468/489] chore(version):upgrade version to v1.49.0 --- .gitignore | 1 + CHANGELOG.md | 5 + src/scanoss/__init__.py | 2 +- tests/data/header-files-test/DataFrame.scala | 71 --- tests/data/header-files-test/FileModel.ts | 152 ----- tests/data/header-files-test/HttpClient.kt | 67 --- tests/data/header-files-test/Package.swift | 70 --- tests/data/header-files-test/Parser.hs | 103 ---- tests/data/header-files-test/Router.php | 83 --- .../data/header-files-test/ServiceProvider.cs | 109 ---- tests/data/header-files-test/StringUtils.hpp | 89 --- .../data/header-files-test/TokenVerifier.java | 538 ------------------ tests/data/header-files-test/ViewController.m | 97 ---- tests/data/header-files-test/analysis.r | 79 --- tests/data/header-files-test/cache.lua | 127 ----- tests/data/header-files-test/config.rs | 103 ---- tests/data/header-files-test/core.clj | 66 --- tests/data/header-files-test/crc32c.c | 421 -------------- tests/data/header-files-test/deploy.sh | 115 ---- tests/data/header-files-test/handler.go | 84 --- tests/data/header-files-test/logger.rb | 73 --- .../header-files-test/multiline_imports.py | 99 ---- tests/data/header-files-test/parser.pl | 94 --- tests/data/header-files-test/results.py | 275 --------- tests/data/header-files-test/server.ex | 100 ---- tests/data/header-files-test/server.js | 90 --- tests/data/header-files-test/widget.dart | 77 --- tests/data/src.tar.gz | Bin 0 -> 29169 bytes 28 files changed, 7 insertions(+), 3183 deletions(-) delete mode 100644 tests/data/header-files-test/DataFrame.scala delete mode 100644 tests/data/header-files-test/FileModel.ts delete mode 100644 tests/data/header-files-test/HttpClient.kt delete mode 100644 tests/data/header-files-test/Package.swift delete mode 100644 tests/data/header-files-test/Parser.hs delete mode 100644 tests/data/header-files-test/Router.php delete mode 100644 tests/data/header-files-test/ServiceProvider.cs delete mode 100644 tests/data/header-files-test/StringUtils.hpp delete mode 100755 tests/data/header-files-test/TokenVerifier.java delete mode 100644 tests/data/header-files-test/ViewController.m delete mode 100644 tests/data/header-files-test/analysis.r delete mode 100644 tests/data/header-files-test/cache.lua delete mode 100644 tests/data/header-files-test/config.rs delete mode 100644 tests/data/header-files-test/core.clj delete mode 100644 tests/data/header-files-test/crc32c.c delete mode 100644 tests/data/header-files-test/deploy.sh delete mode 100644 tests/data/header-files-test/handler.go delete mode 100644 tests/data/header-files-test/logger.rb delete mode 100644 tests/data/header-files-test/multiline_imports.py delete mode 100644 tests/data/header-files-test/parser.pl delete mode 100644 tests/data/header-files-test/results.py delete mode 100644 tests/data/header-files-test/server.ex delete mode 100644 tests/data/header-files-test/server.js delete mode 100644 tests/data/header-files-test/widget.dart create mode 100644 tests/data/src.tar.gz diff --git a/.gitignore b/.gitignore index 2ddf8047..da345dfc 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ docs/build !.devcontainer/*.example.json !tests/data/*.json +!tests/data/header-files-test.zip !docs/source/_static/*.json !scanoss-settings-schema.json .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 517a2f81..a5babe9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.49.0] - 2026-03-11 +### Fixed +- Fixed `--skip-headers` incorrectly identifying continuation lines inside multi-line import blocks + ## [1.48.0] - 2026-03-06 ### Added - Added `--apiurl` option to all component subcommands (`comp vulns`, `comp licenses`, `comp semgrep`, `comp provenance`, `comp search`, `comp versions`) to allow overriding the default API base URL @@ -830,3 +834,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.46.0]: https://github.com/scanoss/scanoss.py/compare/v1.45.1...v1.46.0 [1.47.0]: https://github.com/scanoss/scanoss.py/compare/v1.46.0...v1.47.0 [1.48.0]: https://github.com/scanoss/scanoss.py/compare/v1.47.0...v1.48.0 +[1.49.0]: https://github.com/scanoss/scanoss.py/compare/v1.48.0...v1.49.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 4287241c..352a43ce 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.48.0' +__version__ = '1.49.0' diff --git a/tests/data/header-files-test/DataFrame.scala b/tests/data/header-files-test/DataFrame.scala deleted file mode 100644 index 0b25eef6..00000000 --- a/tests/data/header-files-test/DataFrame.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql - -import scala.collection.mutable.ArrayBuffer -import scala.reflect.runtime.universe.TypeTag - -import org.apache.spark.annotation.{DeveloperApi, Evolving} -import org.apache.spark.api.java.function._ -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.types.StructType - -class DataFrame private[sql]( - @transient val sparkSession: SparkSession, - @DeveloperApi @Evolving val queryExecution: QueryExecution) - extends Dataset[Row] { - - def select(cols: Column*): DataFrame = { - val outputColumns = cols.map(_.named) - sparkSession.sessionState.executePlan( - Project(outputColumns, queryExecution.logical)) - .toDataFrame - } - - def filter(condition: Column): DataFrame = { - sparkSession.sessionState.executePlan( - Filter(condition.expr, queryExecution.logical)) - .toDataFrame - } - - def groupBy(cols: Column*): RelationalGroupedDataset = { - new RelationalGroupedDataset( - this, - cols.map(_.expr), - RelationalGroupedDataset.GroupByType) - } - - def join(right: DataFrame, joinExprs: Column): DataFrame = { - join(right, joinExprs, "inner") - } - - def join(right: DataFrame, joinExprs: Column, joinType: String): DataFrame = { - sparkSession.sessionState.executePlan( - Join(queryExecution.logical, right.queryExecution.logical, - JoinType(joinType), Some(joinExprs.expr), JoinHint.NONE)) - .toDataFrame - } - - def count(): Long = { - groupBy().count().collect().head.getLong(0) - } - - override def toString: String = { - s"DataFrame[${schema.map(f => s"${f.name}: ${f.dataType}").mkString(", ")}]" - } -} \ No newline at end of file diff --git a/tests/data/header-files-test/FileModel.ts b/tests/data/header-files-test/FileModel.ts deleted file mode 100644 index 9f2370b7..00000000 --- a/tests/data/header-files-test/FileModel.ts +++ /dev/null @@ -1,152 +0,0 @@ -import * as util from 'util'; -import sqlite3 from 'sqlite3'; -import { queries } from '../../querys_db'; -import { InventoryModel } from './InventoryModel'; -import { QueryBuilder } from '../../queryBuilder/QueryBuilder'; -import { Model } from '../../Model'; - -const { promisify } = require('util'); - -export class FileModel extends Model { - private connection: sqlite3.Database; - - public static readonly entityMapper = { - path: 'f.path', - purl: 'comp.purl', - version: 'comp.version', - source: 'comp.source', - id: 'fileId', - }; - - inventory: InventoryModel; - - constructor(conn: sqlite3.Database) { - super(); - this.connection = conn; - this.inventory = new InventoryModel(conn); - } - - public async get(queryBuilder: QueryBuilder) { - const SQLquery = this.getSQL( - queryBuilder, - 'SELECT f.fileId, f.path,(CASE WHEN f.identified=1 THEN \'IDENTIFIED\' WHEN f.identified=0 AND f.ignored=0 THEN \'PENDING\' ELSE \'ORIGINAL\' END) AS status, f.type FROM files f #FILTER;', - this.getEntityMapper(), - ); - const call = promisify(this.connection.get.bind(this.connection)); - const file = await call(SQLquery.SQL, ...SQLquery.params); - return file; - } - - public async getAll(queryBuilder?: QueryBuilder): Promise { - const SQLquery = this.getSQL(queryBuilder, queries.SQL_GET_ALL_FILES, this.getEntityMapper()); - const call = promisify(this.connection.all.bind(this.connection)); - const files = await call(SQLquery.SQL, SQLquery.params); - return files; - } - - public async getAllBySearch(queryBuilder?: QueryBuilder): Promise { - const SQLQuery = this.getSQL(queryBuilder, queries.SQL_GET_ALL_FILES_BY_SEARCH, this.getEntityMapper()); - const call:any = util.promisify(this.connection.all.bind(this.connection)); - const files = await call(SQLQuery.SQL, ...SQLQuery.params); - return files; - } - - public async ignored(files: number[]) { - const sql = `${queries.SQL_UPDATE_IGNORED_FILES}(${files.toString()});`; - const call = promisify(this.connection.run.bind(this.connection)); - await call(sql); - } - - public async insertFiles(data: Array) { - return new Promise(async (resolve, reject) => { - this.connection.serialize(async () => { - this.connection.run('begin transaction'); - for (let i = 0; i < data.length; i += 1) { - this.connection.run('INSERT INTO FILES(path,type) VALUES(?,?) ON CONFLICT(path) DO UPDATE SET type = excluded.type; ', data[i].path, data[i].type); - } - - this.connection.run('commit', (err: any) => { - if (!err) resolve(); - reject(err); - }); - }); - }); - } - - public async setDirty(dirty: number, path?: string) { - const call = promisify(this.connection.run.bind(this.connection)); - const SQLquery = path !== undefined ? `UPDATE files SET dirty=${dirty} WHERE path IN (${path});` : `UPDATE files SET dirty=${dirty};`; - await call(SQLquery); - } - - public async getDirty() { - const call = promisify(this.connection.all.bind(this.connection)); - const dirtyFiles = await call('SELECT fileId AS id FROM files WHERE dirty=1;'); - if (dirtyFiles) return dirtyFiles.map((item: any) => item.id); - return []; - } - - public async deleteDirty() { - const call = promisify(this.connection.run.bind(this.connection)); - await call('DELETE FROM files WHERE dirty=1;'); - } - - public async getClean() { - const call = promisify(this.connection.all.bind(this.connection)); - const files = await call('SELECT * FROM files WHERE dirty=0;'); - return files; - } - - public async getFilesRescan() { - const call = promisify(this.connection.all.bind(this.connection)); - const files = await call('SELECT f.path,f.identified ,f.ignored ,f.type AS original,(CASE WHEN f.identified=0 AND f.ignored=0 THEN 1 ELSE 0 END) as pending FROM files f;'); - return files; - } - - public async restore(files: number[]) { - const filesIds = `(${files.toString()});`; - const sql = queries.SQL_FILE_RESTORE + filesIds; - const call = promisify(this.connection.run.bind(this.connection)); - await call(sql); - } - - public async identified(ids: number[]) { - const call = promisify(this.connection.run.bind(this.connection)); - const resultsid = `(${ids.toString()});`; - const sql = queries.SQL_FILES_UPDATE_IDENTIFIED + resultsid; - await call(sql); - } - - public async updateFileType(fileIds: number[], fileType: string) { - const call = promisify(this.connection.run.bind(this.connection)); - const sql = `UPDATE files SET type=? WHERE fileId IN (${fileIds.toString()});`; - await call(sql, fileType); - } - - public async getSummary() { - const call = promisify(this.connection.get.bind(this.connection)); - const summary = await call(`SELECT COUNT(*) as totalFiles , (SELECT COUNT(*) FROM files WHERE type='MATCH') AS matchFiles, - (SELECT COUNT(*) FROM files WHERE type='FILTERED') AS filterFiles, - (SELECT COUNT(*) FROM files WHERE type='NO-MATCH') AS noMatchFiles, (SELECT COUNT(*) FROM files f WHERE f.type="MATCH" AND f.identified=1) AS scannedIdentified, - (SELECT COUNT(*) AS detectedIdentifiedFiles FROM files f WHERE f.identified=1) AS totalIdentified, - (SELECT COUNT(*) AS detectedIdentifiedFiles FROM files f WHERE f.ignored=1) AS original, - (SELECT COUNT(*) AS pending FROM files f WHERE f.identified=0 AND f.ignored=0 AND f.type="MATCH") AS pending FROM files;`); - return summary; - } - - public async getDetectedSummary() { - const call = promisify(this.connection.get.bind(this.connection)); - const summary = await call(`SELECT COUNT(*) as totalFiles , (SELECT COUNT(*) FROM files WHERE type='MATCH') AS matchFiles, - (SELECT COUNT(*) FROM files WHERE type='FILTERED') AS filterFiles, - (SELECT COUNT(*) FROM files WHERE type='NO-MATCH') AS noMatchFiles, - 0 AS scannedIdentified, - 0 AS totalIdentified, - 0 AS original, - (SELECT COUNT(*) AS pending FROM files f WHERE f.identified=0 AND f.ignored=0 AND f.type="MATCH") AS pending FROM files;`); - return summary; - } - - public getEntityMapper(): Record { - return FileModel.entityMapper; - } -} diff --git a/tests/data/header-files-test/HttpClient.kt b/tests/data/header-files-test/HttpClient.kt deleted file mode 100644 index b2cb2015..00000000 --- a/tests/data/header-files-test/HttpClient.kt +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.ktor.client - -import io.ktor.client.engine.* -import io.ktor.client.plugins.* -import io.ktor.client.request.* -import io.ktor.client.statement.* -import io.ktor.util.* -import kotlinx.coroutines.* -import kotlin.coroutines.* - -class HttpClient( - private val engine: HttpClientEngine, - private val config: HttpClientConfig -) : CoroutineScope, Closeable { - - override val coroutineContext: CoroutineContext - get() = engine.coroutineContext - - private val plugins = mutableMapOf, Any>() - - suspend fun request(builder: HttpRequestBuilder): HttpResponse { - val call = engine.execute(builder) - return call.response - } - - suspend fun get(urlString: String, block: HttpRequestBuilder.() -> Unit = {}): HttpResponse { - return request { - url(urlString) - method = HttpMethod.Get - block() - } - } - - suspend fun post(urlString: String, block: HttpRequestBuilder.() -> Unit = {}): HttpResponse { - return request { - url(urlString) - method = HttpMethod.Post - block() - } - } - - fun plugin(key: AttributeKey): T { - @Suppress("UNCHECKED_CAST") - return plugins[key] as? T - ?: throw IllegalStateException("Plugin $key is not installed") - } - - override fun close() { - engine.close() - } -} \ No newline at end of file diff --git a/tests/data/header-files-test/Package.swift b/tests/data/header-files-test/Package.swift deleted file mode 100644 index 18dd0f35..00000000 --- a/tests/data/header-files-test/Package.swift +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors - -import Foundation -import Combine -import os.log - -/// A thread-safe container that manages the lifecycle of registered services. -/// -/// `ServiceContainer` provides dependency injection capabilities by storing -/// factory closures that create service instances on demand. Services can be -/// registered as transient (new instance each time) or singleton (shared instance). -public final class ServiceContainer: @unchecked Sendable { - private let lock = NSRecursiveLock() - private var factories: [String: () -> Any] = [:] - private var singletons: [String: Any] = [:] - private let logger = Logger(subsystem: "com.app", category: "DI") - - public static let shared = ServiceContainer() - - public init() {} - - /// Registers a factory closure for the given type. - public func register(_ type: T.Type, factory: @escaping () -> T) { - let key = String(describing: type) - lock.lock() - defer { lock.unlock() } - factories[key] = factory - logger.debug("Registered service: \(key)") - } - - /// Registers a singleton instance for the given type. - public func registerSingleton(_ type: T.Type, factory: @escaping () -> T) { - let key = String(describing: type) - lock.lock() - defer { lock.unlock() } - singletons[key] = factory() - logger.debug("Registered singleton: \(key)") - } - - /// Resolves an instance of the given type. - public func resolve(_ type: T.Type) -> T? { - let key = String(describing: type) - lock.lock() - defer { lock.unlock() } - - if let singleton = singletons[key] as? T { - return singleton - } - - guard let factory = factories[key] else { - logger.warning("No registration found for: \(key)") - return nil - } - - return factory() as? T - } - - /// Removes all registrations. - public func reset() { - lock.lock() - defer { lock.unlock() } - factories.removeAll() - singletons.removeAll() - logger.info("Container reset") - } -} \ No newline at end of file diff --git a/tests/data/header-files-test/Parser.hs b/tests/data/header-files-test/Parser.hs deleted file mode 100644 index d25847d9..00000000 --- a/tests/data/header-files-test/Parser.hs +++ /dev/null @@ -1,103 +0,0 @@ --- | --- Module : Text.Parsec.Combinator --- Copyright : (c) 2024 Daan Leijen, Paolo Martini --- License : BSD-3-Clause --- --- Maintainer : asr@eafit.edu.co --- Stability : provisional --- Portability : portable --- --- Commonly used generic parser combinators. - -module Text.Parsec.Combinator - ( choice - , count - , between - , option - , optional - , sepBy - , sepBy1 - , many1 - , chainl - , chainl1 - , chainr - , chainr1 - , eof - , notFollowedBy - , manyTill - , lookAhead - ) where - -import Text.Parsec.Prim (Parser, (<|>), try, unexpected, lookAhead) - --- | @choice ps@ tries to apply the parsers in the list @ps@ in order, --- until one of them succeeds. Returns the value of the succeeding parser. -choice :: [Parser a] -> Parser a -choice = foldr (<|>) (unexpected "no match") - --- | @count n p@ parses @n@ occurrences of @p@. -count :: Int -> Parser a -> Parser [a] -count n p - | n <= 0 = return [] - | otherwise = sequence (replicate n p) - --- | @between open close p@ parses @open@, followed by @p@ and @close@. --- Returns the value returned by @p@. -between :: Parser open -> Parser close -> Parser a -> Parser a -between open close p = do - _ <- open - x <- p - _ <- close - return x - --- | @option x p@ tries to apply parser @p@. If @p@ fails without --- consuming input, it returns the value @x@, otherwise the value --- returned by @p@. -option :: a -> Parser a -> Parser a -option x p = p <|> return x - --- | @optional p@ tries to apply parser @p@. It will parse @p@ or nothing. --- It only fails if @p@ fails after consuming input. -optional :: Parser a -> Parser () -optional p = (p >> return ()) <|> return () - --- | @sepBy p sep@ parses zero or more occurrences of @p@, separated --- by @sep@. Returns a list of values returned by @p@. -sepBy :: Parser a -> Parser sep -> Parser [a] -sepBy p sep = sepBy1 p sep <|> return [] - --- | @sepBy1 p sep@ parses one or more occurrences of @p@, separated --- by @sep@. Returns a list of values returned by @p@. -sepBy1 :: Parser a -> Parser sep -> Parser [a] -sepBy1 p sep = do - x <- p - xs <- many (sep >> p) - return (x : xs) - --- | @many1 p@ applies the parser @p@ one or more times. -many1 :: Parser a -> Parser [a] -many1 p = do - x <- p - xs <- many p - return (x : xs) - --- | @chainl p op x@ parses zero or more occurrences of @p@, --- separated by @op@. Returns a value obtained by a left associative --- application of all functions returned by @op@ to the values --- returned by @p@. If there are zero occurrences of @p@, the value --- @x@ is returned. -chainl :: Parser a -> Parser (a -> a -> a) -> a -> Parser a -chainl p op x = chainl1 p op <|> return x - --- | @chainl1 p op@ parses one or more occurrences of @p@, --- separated by @op@. Returns a value obtained by a left associative --- application of all functions returned by @op@. -chainl1 :: Parser a -> Parser (a -> a -> a) -> Parser a -chainl1 p op = do - x <- p - rest x - where - rest x = (do f <- op - y <- p - rest (f x y)) - <|> return x \ No newline at end of file diff --git a/tests/data/header-files-test/Router.php b/tests/data/header-files-test/Router.php deleted file mode 100644 index 9fb92a60..00000000 --- a/tests/data/header-files-test/Router.php +++ /dev/null @@ -1,83 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Symfony\Component\Routing; - -use Symfony\Component\HttpFoundation\Request; -use Symfony\Component\Routing\Exception\MethodNotAllowedException; -use Symfony\Component\Routing\Exception\ResourceNotFoundException; -use Symfony\Component\Routing\Matcher\UrlMatcherInterface; - -class Router implements RouterInterface -{ - private RouteCollection $routes; - private UrlMatcherInterface $matcher; - private array $options; - - public function __construct(RouteCollection $routes, array $options = []) - { - $this->routes = $routes; - $this->options = array_merge([ - 'cache_dir' => null, - 'debug' => false, - 'strict_requirements' => true, - ], $options); - } - - public function match(string $pathinfo): array - { - return $this->getMatcher()->match($pathinfo); - } - - public function matchRequest(Request $request): array - { - $pathinfo = $request->getPathInfo(); - $method = $request->getMethod(); - - try { - $parameters = $this->match($pathinfo); - } catch (ResourceNotFoundException $e) { - throw new ResourceNotFoundException( - sprintf('No route found for "%s %s"', $method, $pathinfo), - 0, - $e - ); - } - - if (isset($parameters['_method'])) { - $allowedMethods = explode('|', $parameters['_method']); - if (!in_array($method, $allowedMethods, true)) { - throw new MethodNotAllowedException($allowedMethods); - } - } - - return $parameters; - } - - public function getRouteCollection(): RouteCollection - { - return $this->routes; - } - - public function addRoute(string $name, Route $route): void - { - $this->routes->add($name, $route); - } - - private function getMatcher(): UrlMatcherInterface - { - if (!isset($this->matcher)) { - $this->matcher = new UrlMatcher($this->routes, new RequestContext()); - } - - return $this->matcher; - } -} \ No newline at end of file diff --git a/tests/data/header-files-test/ServiceProvider.cs b/tests/data/header-files-test/ServiceProvider.cs deleted file mode 100644 index f68f38c3..00000000 --- a/tests/data/header-files-test/ServiceProvider.cs +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; - -namespace Microsoft.Extensions.DependencyInjection -{ - /// - /// The default IServiceProvider implementation. - /// - public sealed class ServiceProvider : IServiceProvider, IDisposable, IAsyncDisposable - { - private readonly ConcurrentDictionary> _factories; - private readonly ConcurrentDictionary _singletons; - private readonly IServiceCollection _services; - private bool _disposed; - - internal ServiceProvider(IServiceCollection services) - { - _services = services ?? throw new ArgumentNullException(nameof(services)); - _factories = new ConcurrentDictionary>(); - _singletons = new ConcurrentDictionary(); - } - - /// - /// Gets the service object of the specified type. - /// - public object GetService(Type serviceType) - { - if (_disposed) - throw new ObjectDisposedException(nameof(ServiceProvider)); - - if (serviceType == typeof(IServiceProvider)) - return this; - - if (_singletons.TryGetValue(serviceType, out var singleton)) - return singleton; - - if (_factories.TryGetValue(serviceType, out var factory)) - return factory(this); - - var descriptor = _services.FirstOrDefault(d => d.ServiceType == serviceType); - if (descriptor == null) - return null; - - return CreateInstance(descriptor); - } - - private object CreateInstance(ServiceDescriptor descriptor) - { - if (descriptor.ImplementationInstance != null) - return descriptor.ImplementationInstance; - - if (descriptor.ImplementationFactory != null) - return descriptor.ImplementationFactory(this); - - var implementationType = descriptor.ImplementationType; - var constructor = implementationType.GetConstructors().OrderByDescending(c => c.GetParameters().Length).First(); - - var parameters = constructor.GetParameters() - .Select(p => GetService(p.ParameterType)) - .ToArray(); - - return constructor.Invoke(parameters); - } - - public void Dispose() - { - if (_disposed) - return; - - _disposed = true; - - foreach (var singleton in _singletons.Values) - { - if (singleton is IDisposable disposable) - disposable.Dispose(); - } - - _singletons.Clear(); - _factories.Clear(); - } - - public async ValueTask DisposeAsync() - { - if (_disposed) - return; - - _disposed = true; - - foreach (var singleton in _singletons.Values) - { - if (singleton is IAsyncDisposable asyncDisposable) - await asyncDisposable.DisposeAsync(); - else if (singleton is IDisposable disposable) - disposable.Dispose(); - } - - _singletons.Clear(); - _factories.Clear(); - } - } -} \ No newline at end of file diff --git a/tests/data/header-files-test/StringUtils.hpp b/tests/data/header-files-test/StringUtils.hpp deleted file mode 100644 index 9a401900..00000000 --- a/tests/data/header-files-test/StringUtils.hpp +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright (c) 2024 LLVM Project Contributors -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. - -#ifndef LLVM_ADT_STRING_UTILS_HPP -#define LLVM_ADT_STRING_UTILS_HPP - -#include -#include -#include -#include -#include -#include - -namespace llvm { - -/// Splits a string by the given delimiter and returns a vector of substrings. -/// -/// \param input The string to split. -/// \param delimiter The character to split on. -/// \return A vector of substrings. -inline std::vector split(const std::string &input, char delimiter) { - std::vector tokens; - std::istringstream stream(input); - std::string token; - - while (std::getline(stream, token, delimiter)) { - if (!token.empty()) { - tokens.push_back(token); - } - } - return tokens; -} - -/// Trims whitespace from the beginning and end of a string. -inline std::string trim(const std::string &str) { - auto start = std::find_if_not(str.begin(), str.end(), ::isspace); - auto end = std::find_if_not(str.rbegin(), str.rend(), ::isspace).base(); - - if (start >= end) { - return ""; - } - return std::string(start, end); -} - -/// Converts a string to lowercase. -inline std::string toLower(const std::string &str) { - std::string result = str; - std::transform(result.begin(), result.end(), result.begin(), ::tolower); - return result; -} - -/// Converts a string to uppercase. -inline std::string toUpper(const std::string &str) { - std::string result = str; - std::transform(result.begin(), result.end(), result.begin(), ::toupper); - return result; -} - -/// Checks if a string starts with the given prefix. -inline bool startsWith(const std::string &str, const std::string &prefix) { - return str.size() >= prefix.size() && - str.compare(0, prefix.size(), prefix) == 0; -} - -/// Checks if a string ends with the given suffix. -inline bool endsWith(const std::string &str, const std::string &suffix) { - return str.size() >= suffix.size() && - str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; -} - -/// Replaces all occurrences of a substring with another substring. -inline std::string replaceAll(const std::string &str, - const std::string &from, - const std::string &to) { - std::string result = str; - size_t pos = 0; - while ((pos = result.find(from, pos)) != std::string::npos) { - result.replace(pos, from.length(), to); - pos += to.length(); - } - return result; -} - -} // namespace llvm - -#endif // LLVM_ADT_STRING_UTILS_HPP \ No newline at end of file diff --git a/tests/data/header-files-test/TokenVerifier.java b/tests/data/header-files-test/TokenVerifier.java deleted file mode 100755 index 90b6e3e1..00000000 --- a/tests/data/header-files-test/TokenVerifier.java +++ /dev/null @@ -1,538 +0,0 @@ -/* - * Copyright 2016 Red Hat, Inc. and/or its affiliates - * and other contributors as indicated by the @author tags. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.keycloak; - -import java.nio.charset.StandardCharsets; -import java.security.PublicKey; -import java.util.Arrays; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.crypto.SecretKey; - -import org.keycloak.common.VerificationException; -import org.keycloak.crypto.SignatureVerifierContext; -import org.keycloak.exceptions.TokenNotActiveException; -import org.keycloak.exceptions.TokenSignatureInvalidException; -import org.keycloak.jose.jws.AlgorithmType; -import org.keycloak.jose.jws.JWSHeader; -import org.keycloak.jose.jws.JWSInput; -import org.keycloak.jose.jws.JWSInputException; -import org.keycloak.jose.jws.crypto.HMACProvider; -import org.keycloak.jose.jws.crypto.RSAProvider; -import org.keycloak.representations.JsonWebToken; -import org.keycloak.util.TokenUtil; - -/** - * @author Bill Burke - * @version $Revision: 1 $ - */ -public class TokenVerifier { - - private static final Logger LOG = Logger.getLogger(TokenVerifier.class.getName()); - - // This interface is here as JDK 7 is a requirement for this project. - // Once JDK 8 would become mandatory, java.util.function.Predicate would be used instead. - - /** - * Functional interface of checks that verify some part of a JWT. - * @param Type of the token handled by this predicate. - */ - // @FunctionalInterface - public static interface Predicate { - /** - * Performs a single check on the given token verifier. - * @param t Token, guaranteed to be non-null. - * @return - * @throws VerificationException - */ - boolean test(T t) throws VerificationException; - } - - public static final Predicate SUBJECT_EXISTS_CHECK = new Predicate() { - @Override - public boolean test(JsonWebToken t) throws VerificationException { - String subject = t.getSubject(); - if (subject == null) { - throw new VerificationException("Subject missing in token"); - } - - return true; - } - }; - - /** - * Check for token being neither expired nor used before it gets valid. - * @see JsonWebToken#isActive() - */ - public static final Predicate IS_ACTIVE = new Predicate() { - @Override - public boolean test(JsonWebToken t) throws VerificationException { - if (! t.isActive()) { - throw new TokenNotActiveException(t, "Token is not active"); - } - - return true; - } - }; - - public static class RealmUrlCheck implements Predicate { - - private static final RealmUrlCheck NULL_INSTANCE = new RealmUrlCheck(null); - - private final String realmUrl; - - public RealmUrlCheck(String realmUrl) { - this.realmUrl = realmUrl; - } - - @Override - public boolean test(JsonWebToken t) throws VerificationException { - if (this.realmUrl == null) { - throw new VerificationException("Realm URL not set"); - } - - if (! this.realmUrl.equals(t.getIssuer())) { - throw new VerificationException("Invalid token issuer. Expected '" + this.realmUrl + "'"); - } - - return true; - } - } - - public static class TokenTypeCheck implements Predicate { - - private static final TokenTypeCheck INSTANCE_DEFAULT_TOKEN_TYPE = new TokenTypeCheck(Arrays.asList(TokenUtil.TOKEN_TYPE_BEARER)); - - private final List tokenTypes; - - public TokenTypeCheck(List tokenTypes) { - this.tokenTypes = tokenTypes; - } - - @Override - public boolean test(JsonWebToken t) throws VerificationException { - for (String tokenType : tokenTypes) { - if (tokenType.equalsIgnoreCase(t.getType())) return true; - } - throw new VerificationException("Token type is incorrect. Expected '" + tokenTypes.toString() + "' but was '" + t.getType() + "'"); - } - } - - - public static class AudienceCheck implements Predicate { - - private final String expectedAudience; - - public AudienceCheck(String expectedAudience) { - this.expectedAudience = expectedAudience; - } - - @Override - public boolean test(JsonWebToken t) throws VerificationException { - if (expectedAudience == null) { - throw new VerificationException("Missing expectedAudience"); - } - - String[] audience = t.getAudience(); - if (audience == null) { - throw new VerificationException("No audience in the token"); - } - - if (t.hasAudience(expectedAudience)) { - return true; - } - - throw new VerificationException("Expected audience not available in the token"); - } - } - - - public static class IssuedForCheck implements Predicate { - - private final String expectedIssuedFor; - - public IssuedForCheck(String expectedIssuedFor) { - this.expectedIssuedFor = expectedIssuedFor; - } - - @Override - public boolean test(JsonWebToken jsonWebToken) throws VerificationException { - if (expectedIssuedFor == null) { - throw new VerificationException("Missing expectedIssuedFor"); - } - - if (expectedIssuedFor.equals(jsonWebToken.getIssuedFor())) { - return true; - } - - throw new VerificationException("Expected issuedFor doesn't match"); - } - } - - - private String tokenString; - private Class clazz; - private PublicKey publicKey; - private SecretKey secretKey; - private String realmUrl; - private List expectedTokenType = Arrays.asList(TokenUtil.TOKEN_TYPE_BEARER, TokenUtil.TOKEN_TYPE_DPOP); - private boolean checkTokenType = true; - private boolean checkRealmUrl = true; - private final LinkedList> checks = new LinkedList<>(); - - private JWSInput jws; - private T token; - - private SignatureVerifierContext verifier = null; - - public TokenVerifier verifierContext(SignatureVerifierContext verifier) { - this.verifier = verifier; - return this; - } - - protected TokenVerifier(String tokenString, Class clazz) { - this.tokenString = tokenString; - this.clazz = clazz; - } - - protected TokenVerifier(T token) { - this.token = token; - } - - /** - * Creates an instance of {@code TokenVerifier} from the given string on a JWT of the given class. - * The token verifier has no checks defined. Note that the checks are only tested when - * {@link #verify()} method is invoked. - * @param Type of the token - * @param tokenString String representation of JWT - * @param clazz Class of the token - * @return - */ - public static TokenVerifier create(String tokenString, Class clazz) { - return new TokenVerifier<>(tokenString, clazz); - } - - /** - * Creates an instance of {@code TokenVerifier} for the given token. - * The token verifier has no checks defined. Note that the checks are only tested when - * {@link #verify()} method is invoked. - *

- * NOTE: The returned token verifier cannot verify token signature since - * that is not part of the {@link JsonWebToken} object. - * @return - */ - public static TokenVerifier createWithoutSignature(T token) { - return new TokenVerifier<>(token); - } - - /** - * Adds default checks to the token verification: - *

    - *
  • Realm URL (JWT issuer field: {@code iss}) has to be defined and match realm set via {@link #realmUrl(java.lang.String)} method
  • - *
  • Subject (JWT subject field: {@code sub}) has to be defined
  • - *
  • Token type (JWT type field: {@code typ}) has to be {@code Bearer}. The type can be set via {@link #tokenType(List)} method
  • - *
  • Token has to be active, ie. both not expired and not used before its validity (JWT issuer fields: {@code exp} and {@code nbf})
  • - *
- * @return This token verifier. - */ - public TokenVerifier withDefaultChecks() { - return withChecks( - RealmUrlCheck.NULL_INSTANCE, - TokenTypeCheck.INSTANCE_DEFAULT_TOKEN_TYPE, - IS_ACTIVE - ); - } - - private void removeCheck(Class> checkClass) { - for (Iterator> it = checks.iterator(); it.hasNext();) { - if (it.next().getClass() == checkClass) { - it.remove(); - } - } - } - - private void removeCheck(Predicate check) { - checks.remove(check); - } - - @SuppressWarnings("unchecked") - private

> TokenVerifier replaceCheck(Class> checkClass, boolean active, P... predicate) { - removeCheck(checkClass); - if (active) { - checks.addAll(Arrays.asList(predicate)); - } - return this; - } - - @SuppressWarnings("unchecked") - private

> TokenVerifier replaceCheck(Predicate check, boolean active, P... predicate) { - removeCheck(check); - if (active) { - checks.addAll(Arrays.asList(predicate)); - } - return this; - } - - /** - * Will test the given checks in {@link #verify()} method in addition to already set checks. - * @param checks - * @return - */ - @SafeVarargs - public final TokenVerifier withChecks(Predicate... checks) { - if (checks != null) { - this.checks.addAll(Arrays.asList(checks)); - } - return this; - } - - /** - * Sets the key for verification of RSA-based signature. - * @param publicKey - * @return - */ - public TokenVerifier publicKey(PublicKey publicKey) { - this.publicKey = publicKey; - return this; - } - - /** - * Sets the key for verification of HMAC-based signature. - * @param secretKey - * @return - */ - public TokenVerifier secretKey(SecretKey secretKey) { - this.secretKey = secretKey; - return this; - } - - /** - * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. - * @return This token verifier - */ - public TokenVerifier realmUrl(String realmUrl) { - this.realmUrl = realmUrl; - return replaceCheck(RealmUrlCheck.class, checkRealmUrl, new RealmUrlCheck(realmUrl)); - } - - /** - * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. - * @return This token verifier - */ - public TokenVerifier checkTokenType(boolean checkTokenType) { - this.checkTokenType = checkTokenType; - return replaceCheck(TokenTypeCheck.class, this.checkTokenType, new TokenTypeCheck(expectedTokenType)); - } - - /** - * - * @return This token verifier - */ - public TokenVerifier tokenType(List tokenTypes) { - this.expectedTokenType = tokenTypes; - return replaceCheck(TokenTypeCheck.class, this.checkTokenType, new TokenTypeCheck(expectedTokenType)); - } - - /** - * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. - * @return This token verifier - */ - public TokenVerifier checkActive(boolean checkActive) { - return replaceCheck(IS_ACTIVE, checkActive, IS_ACTIVE); - } - - /** - * @deprecated This method is here only for backward compatibility with previous version of {@code TokenVerifier}. - * @return This token verifier - */ - public TokenVerifier checkRealmUrl(boolean checkRealmUrl) { - this.checkRealmUrl = checkRealmUrl; - return replaceCheck(RealmUrlCheck.class, this.checkRealmUrl, new RealmUrlCheck(realmUrl)); - } - - /** - * Add check for verifying that token contains the expectedAudience - * - * @param expectedAudiences Audiences, which needs to be in the target token. Can be null. - * @return This token verifier - */ - public TokenVerifier audience(String... expectedAudiences) { - if (expectedAudiences == null || expectedAudiences.length == 0) { - return this.replaceCheck(AudienceCheck.class, true, new AudienceCheck(null)); - } - AudienceCheck[] audienceChecks = new AudienceCheck[expectedAudiences.length]; - for (int i = 0; i < expectedAudiences.length; ++i) { - audienceChecks[i] = new AudienceCheck(expectedAudiences[i]); - } - return this.replaceCheck(AudienceCheck.class, true, audienceChecks); - } - - /** - * Add check for verifying that token issuedFor (azp claim) is the expected value - * - * @param expectedIssuedFor issuedFor, which needs to be in the target token. Can't be null - * @return This token verifier - */ - public TokenVerifier issuedFor(String expectedIssuedFor) { - return this.replaceCheck(IssuedForCheck.class, true, new IssuedForCheck(expectedIssuedFor)); - } - - public TokenVerifier parse() throws VerificationException { - if (jws == null) { - if (tokenString == null) { - throw new VerificationException("Token not set"); - } - - try { - jws = new JWSInput(tokenString); - } catch (JWSInputException e) { - throw new VerificationException("Failed to parse JWT", e); - } - - - try { - token = jws.readJsonContent(clazz); - } catch (JWSInputException e) { - throw new VerificationException("Failed to read access token from JWT", e); - } - } - return this; - } - - public T getToken() throws VerificationException { - if (token == null) { - parse(); - } - return token; - } - - public JWSHeader getHeader() throws VerificationException { - parse(); - return jws.getHeader(); - } - - public void verifySignature() throws VerificationException { - if (this.verifier != null) { - try { - if (!verifier.verify(jws.getEncodedSignatureInput().getBytes(StandardCharsets.UTF_8), jws.getSignature())) { - throw new TokenSignatureInvalidException(token, "Invalid token signature"); - } - } catch (Exception e) { - throw new VerificationException(e); - } - } else { - AlgorithmType algorithmType = getHeader().getAlgorithm().getType(); - - if (null == algorithmType) { - throw new VerificationException("Unknown or unsupported token algorithm"); - } else switch (algorithmType) { - case RSA: - if (publicKey == null) { - throw new VerificationException("Public key not set"); - } - if (!RSAProvider.verify(jws, publicKey)) { - throw new TokenSignatureInvalidException(token, "Invalid token signature"); - } - break; - case HMAC: - if (secretKey == null) { - throw new VerificationException("Secret key not set"); - } - if (!HMACProvider.verify(jws, secretKey)) { - throw new TokenSignatureInvalidException(token, "Invalid token signature"); - } - break; - default: - throw new VerificationException("Unknown or unsupported token algorithm"); - } - } - } - - public TokenVerifier verify() throws VerificationException { - if (getToken() == null) { - parse(); - } - if (jws != null) { - verifySignature(); - } - - for (Predicate check : checks) { - if (! check.test(getToken())) { - throw new VerificationException("JWT check failed for check " + check); - } - } - - return this; - } - - /** - * Creates an optional predicate from a predicate that will proceed with check but always pass. - * @param - * @param mandatoryPredicate - * @return - */ - public static Predicate optional(final Predicate mandatoryPredicate) { - return new Predicate() { - @Override - public boolean test(T t) throws VerificationException { - try { - if (! mandatoryPredicate.test(t)) { - LOG.finer("[optional] predicate failed: " + mandatoryPredicate); - } - - return true; - } catch (VerificationException ex) { - LOG.log(Level.FINER, "[optional] predicate " + mandatoryPredicate + " failed.", ex); - return true; - } - } - }; - } - - /** - * Creates a predicate that will proceed with checks of the given predicates - * and will pass if and only if at least one of the given predicates passes. - * @param - * @param predicates - * @return - */ - @SafeVarargs - public static Predicate alternative(final Predicate... predicates) { - return new Predicate() { - @Override - public boolean test(T t) { - for (Predicate predicate : predicates) { - try { - if (predicate.test(t)) { - return true; - } - - LOG.finer("[alternative] predicate failed: " + predicate); - } catch (VerificationException ex) { - LOG.log(Level.FINER, "[alternative] predicate " + predicate + " failed.", ex); - } - } - - return false; - } - }; - } -} diff --git a/tests/data/header-files-test/ViewController.m b/tests/data/header-files-test/ViewController.m deleted file mode 100644 index 16b9090c..00000000 --- a/tests/data/header-files-test/ViewController.m +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2024 Apple Inc. -// Licensed under the Apache License, Version 2.0. -// See LICENSE file in the project root for full license information. -// -// SPDX-License-Identifier: Apache-2.0 - -#import "ViewController.h" -#import -#import - -@interface ViewController () - -@property (nonatomic, strong) UITableView *tableView; -@property (nonatomic, strong) NSMutableArray *dataSource; -@property (nonatomic, strong) UIRefreshControl *refreshControl; - -@end - -@implementation ViewController - -- (void)viewDidLoad { - [super viewDidLoad]; - self.title = @"Items"; - self.dataSource = [NSMutableArray array]; - - [self setupTableView]; - [self setupRefreshControl]; - [self loadData]; -} - -- (void)setupTableView { - self.tableView = [[UITableView alloc] initWithFrame:self.view.bounds - style:UITableViewStylePlain]; - self.tableView.dataSource = self; - self.tableView.delegate = self; - self.tableView.autoresizingMask = UIViewAutoresizingFlexibleWidth | - UIViewAutoresizingFlexibleHeight; - [self.tableView registerClass:[UITableViewCell class] - forCellReuseIdentifier:@"Cell"]; - [self.view addSubview:self.tableView]; -} - -- (void)setupRefreshControl { - self.refreshControl = [[UIRefreshControl alloc] init]; - [self.refreshControl addTarget:self - action:@selector(refreshData) - forControlEvents:UIControlEventValueChanged]; - self.tableView.refreshControl = self.refreshControl; -} - -- (void)loadData { - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - NSArray *items = @[ - @{@"title": @"First Item", @"subtitle": @"Description 1"}, - @{@"title": @"Second Item", @"subtitle": @"Description 2"}, - @{@"title": @"Third Item", @"subtitle": @"Description 3"}, - ]; - - dispatch_async(dispatch_get_main_queue(), ^{ - [self.dataSource removeAllObjects]; - [self.dataSource addObjectsFromArray:items]; - [self.tableView reloadData]; - [self.refreshControl endRefreshing]; - }); - }); -} - -- (void)refreshData { - [self loadData]; -} - -#pragma mark - UITableViewDataSource - -- (NSInteger)tableView:(UITableView *)tableView numberOfRowsInSection:(NSInteger)section { - return self.dataSource.count; -} - -- (UITableViewCell *)tableView:(UITableView *)tableView - cellForRowAtIndexPath:(NSIndexPath *)indexPath { - UITableViewCell *cell = [tableView dequeueReusableCellWithIdentifier:@"Cell" - forIndexPath:indexPath]; - NSDictionary *item = self.dataSource[indexPath.row]; - cell.textLabel.text = item[@"title"]; - cell.detailTextLabel.text = item[@"subtitle"]; - cell.accessoryType = UITableViewCellAccessoryDisclosureIndicator; - return cell; -} - -#pragma mark - UITableViewDelegate - -- (void)tableView:(UITableView *)tableView didSelectRowAtIndexPath:(NSIndexPath *)indexPath { - [tableView deselectRowAtIndexPath:indexPath animated:YES]; - NSDictionary *item = self.dataSource[indexPath.row]; - NSLog(@"Selected: %@", item[@"title"]); -} - -@end \ No newline at end of file diff --git a/tests/data/header-files-test/analysis.r b/tests/data/header-files-test/analysis.r deleted file mode 100644 index c2a7bc71..00000000 --- a/tests/data/header-files-test/analysis.r +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2024 The R Foundation for Statistical Computing -# -# This file is part of R, which is free software. You can redistribute it -# and/or modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# R is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. - -library(ggplot2) -library(dplyr) -library(tidyr) -require(stats) - -#' Perform statistical analysis on the given dataset -#' -#' @param data A data frame containing the input data -#' @param target_col The name of the target variable column -#' @param predictor_cols A character vector of predictor column names -#' @return A list containing model summary and diagnostics -analyze_regression <- function(data, target_col, predictor_cols) { - if (!is.data.frame(data)) { - stop("Input must be a data frame") - } - - formula_str <- paste(target_col, "~", paste(predictor_cols, collapse = " + ")) - model <- lm(as.formula(formula_str), data = data) - - residuals <- residuals(model) - fitted_vals <- fitted(model) - - diagnostics <- list( - shapiro_test = shapiro.test(residuals), - r_squared = summary(model)$r.squared, - adj_r_squared = summary(model)$adj.r.squared, - f_statistic = summary(model)$fstatistic - ) - - result <- list( - model = model, - summary = summary(model), - diagnostics = diagnostics - ) - - return(result) -} - -plot_diagnostics <- function(model, output_dir = "plots") { - if (!dir.exists(output_dir)) { - dir.create(output_dir, recursive = TRUE) - } - - residual_data <- data.frame( - fitted = fitted(model), - residuals = residuals(model), - standardized = rstandard(model) - ) - - p1 <- ggplot(residual_data, aes(x = fitted, y = residuals)) + - geom_point(alpha = 0.5) + - geom_hline(yintercept = 0, linetype = "dashed", color = "red") + - labs(title = "Residuals vs Fitted", x = "Fitted Values", y = "Residuals") + - theme_minimal() - - ggsave(file.path(output_dir, "residuals_vs_fitted.png"), p1) - - p2 <- ggplot(residual_data, aes(sample = standardized)) + - stat_qq() + - stat_qq_line(color = "red") + - labs(title = "Normal Q-Q Plot") + - theme_minimal() - - ggsave(file.path(output_dir, "qq_plot.png"), p2) - - invisible(list(p1, p2)) -} \ No newline at end of file diff --git a/tests/data/header-files-test/cache.lua b/tests/data/header-files-test/cache.lua deleted file mode 100644 index 4f611936..00000000 --- a/tests/data/header-files-test/cache.lua +++ /dev/null @@ -1,127 +0,0 @@ --- Copyright (c) 2024 OpenResty Inc. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -local require = require -local setmetatable = setmetatable -local ngx = ngx -local type = type -local error = error -local tostring = tostring -local math_floor = math.floor -local os_time = os.time - -local _M = {} -local mt = { __index = _M } - -_M._VERSION = '0.1.0' - -function _M.new(max_size, default_ttl) - if type(max_size) ~= "number" or max_size < 1 then - error("max_size must be a positive number") - end - - local self = { - store = {}, - expiry = {}, - max_size = max_size, - size = 0, - default_ttl = default_ttl or 300, - hits = 0, - misses = 0, - } - - return setmetatable(self, mt) -end - -function _M.set(self, key, value, ttl) - if key == nil then - return nil, "key is nil" - end - - ttl = ttl or self.default_ttl - - if self.store[key] == nil then - if self.size >= self.max_size then - self:_evict() - end - self.size = self.size + 1 - end - - self.store[key] = value - self.expiry[key] = os_time() + ttl - - return true -end - -function _M.get(self, key) - local value = self.store[key] - if value == nil then - self.misses = self.misses + 1 - return nil, "not found" - end - - local exp = self.expiry[key] - if exp and os_time() > exp then - self:delete(key) - self.misses = self.misses + 1 - return nil, "expired" - end - - self.hits = self.hits + 1 - return value -end - -function _M.delete(self, key) - if self.store[key] ~= nil then - self.store[key] = nil - self.expiry[key] = nil - self.size = self.size - 1 - return true - end - return false -end - -function _M._evict(self) - local oldest_key = nil - local oldest_time = nil - - for key, exp in pairs(self.expiry) do - if oldest_time == nil or exp < oldest_time then - oldest_key = key - oldest_time = exp - end - end - - if oldest_key then - self:delete(oldest_key) - end -end - -function _M.flush(self) - self.store = {} - self.expiry = {} - self.size = 0 -end - -function _M.stats(self) - return { - size = self.size, - max_size = self.max_size, - hits = self.hits, - misses = self.misses, - hit_rate = self.hits / (self.hits + self.misses + 1), - } -end - -return _M \ No newline at end of file diff --git a/tests/data/header-files-test/config.rs b/tests/data/header-files-test/config.rs deleted file mode 100644 index 4db5e7d7..00000000 --- a/tests/data/header-files-test/config.rs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2024 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use std::collections::HashMap; -use std::fs; -use std::io::{self, Read}; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; - -/// Configuration for the application. -/// -/// This struct holds all the configuration parameters that can be -/// loaded from a TOML configuration file or set via environment -/// variables. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Config { - pub server: ServerConfig, - pub database: DatabaseConfig, - pub logging: LoggingConfig, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ServerConfig { - pub host: String, - pub port: u16, - pub workers: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DatabaseConfig { - pub url: String, - pub max_connections: u32, - pub min_connections: u32, - pub connection_timeout: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LoggingConfig { - pub level: String, - pub file: Option, - pub format: String, -} - -impl Config { - /// Load configuration from the specified file path. - pub fn from_file(path: &Path) -> io::Result { - let mut file = fs::File::open(path)?; - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - - toml::from_str(&contents) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) - } - - /// Load configuration with environment variable overrides. - pub fn from_env() -> Self { - let mut config = Self::default(); - - if let Ok(host) = std::env::var("SERVER_HOST") { - config.server.host = host; - } - if let Ok(port) = std::env::var("SERVER_PORT") { - if let Ok(port) = port.parse() { - config.server.port = port; - } - } - if let Ok(url) = std::env::var("DATABASE_URL") { - config.database.url = url; - } - - config - } -} - -impl Default for Config { - fn default() -> Self { - Config { - server: ServerConfig { - host: "127.0.0.1".to_string(), - port: 8080, - workers: num_cpus::get(), - }, - database: DatabaseConfig { - url: "postgres://localhost/mydb".to_string(), - max_connections: 10, - min_connections: 1, - connection_timeout: 30, - }, - logging: LoggingConfig { - level: "info".to_string(), - file: None, - format: "pretty".to_string(), - }, - } - } -} \ No newline at end of file diff --git a/tests/data/header-files-test/core.clj b/tests/data/header-files-test/core.clj deleted file mode 100644 index 3303ca0d..00000000 --- a/tests/data/header-files-test/core.clj +++ /dev/null @@ -1,66 +0,0 @@ -;; Copyright (c) Rich Hickey. All rights reserved. -;; The use and distribution terms for this software are covered by the -;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) -;; which can be found in the file epl-v10.html at the root of this distribution. -;; By using this software in any fashion, you are agreeing to be bound by -;; the terms of this license. -;; You must not remove this notice, or any other, from this software. - -(ns myapp.core - (:require [clojure.string :as str] - [clojure.java.io :as io] - [clojure.edn :as edn] - [clojure.tools.logging :as log])) - -(defn load-config - "Load configuration from an EDN file. - Returns a map of configuration values." - [path] - (try - (with-open [reader (io/reader path)] - (edn/read (java.io.PushbackReader. reader))) - (catch Exception e - (log/error e "Failed to load config from" path) - {}))) - -(defn parse-request - "Parse an HTTP request string into a map." - [request-str] - (let [lines (str/split-lines request-str) - [method path version] (str/split (first lines) #"\s+") - headers (->> (rest lines) - (take-while (complement str/blank?)) - (map #(str/split % #":\s*" 2)) - (filter #(= 2 (count %))) - (into {} (map (fn [[k v]] [(str/lower-case k) v]))))] - {:method method - :path path - :version version - :headers headers})) - -(defn route-request - "Route a request to the appropriate handler." - [routes request] - (let [handler (get-in routes [(:method request) (:path request)])] - (if handler - (handler request) - {:status 404 - :body "Not Found"}))) - -(defn start-server - "Start the application server with the given configuration." - [config] - (let [port (get config :port 8080) - host (get config :host "0.0.0.0")] - (log/info "Starting server on" host ":" port) - {:port port - :host host - :status :running})) - -(defn -main - "Application entry point." - [& args] - (let [config-path (or (first args) "config.edn") - config (load-config config-path)] - (log/info "Loaded configuration:" config) - (start-server config))) \ No newline at end of file diff --git a/tests/data/header-files-test/crc32c.c b/tests/data/header-files-test/crc32c.c deleted file mode 100644 index 6edadd2f..00000000 --- a/tests/data/header-files-test/crc32c.c +++ /dev/null @@ -1,421 +0,0 @@ -// SPDX-License-Identifier: Zlib -/* crc32c.c -- compute CRC-32C using the Intel crc32 instruction - * Copyright (C) 2013 Mark Adler - * Version 1.1 1 Aug 2013 Mark Adler - */ - -/* - This software is provided 'as-is', without any express or implied - warranty. In no event will the author be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Mark Adler - madler@alumni.caltech.edu - */ - -/** - @file crc32c.c - @date 14 Dec 2020 - @brief Use hardware CRC instruction on Intel SSE 4.2 processors. - This computes a - CRC-32C, *not* the CRC-32 used by Ethernet and zip, gzip, etc. A software - version is provided as a fall-back, as well as for speed comparisons. - @see https://github.com/scanoss/engine/blob/master/external/src/crc32c.c - */ - -/* Version history: - 1.0 10 Feb 2013 First version - 1.1 1 Aug 2013 Correct comments on why three crc instructions in parallel - */ - -#include -#include -#include -#include -#include -#include - -/* CRC-32C (iSCSI) polynomial in reversed bit order. */ -#define POLY 0x82f63b78 - -/* Table for a quadword-at-a-time software crc. */ -static pthread_once_t crc32c_once_sw = PTHREAD_ONCE_INIT; -static uint32_t crc32c_table[8][256]; - -/** - * @brief Construct table for software CRC-32C calculation. - */ -static void crc32c_init_sw(void) -{ - uint32_t n, crc, k; - - for (n = 0; n < 256; n++) { - crc = n; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1; - crc32c_table[0][n] = crc; - } - for (n = 0; n < 256; n++) { - crc = crc32c_table[0][n]; - for (k = 1; k < 8; k++) { - crc = crc32c_table[0][crc & 0xff] ^ (crc >> 8); - crc32c_table[k][n] = crc; - } - } -} - -/** - * @brief Table-driven software version as a fall-back. This is about 15 times slower - than using the hardware instructions. This assumes little-endian integers, - as is the case on Intel processors that the assembler code here is for. - * @param crci Acummulated valued - * @param buf Data buffer - * @param len Data buffer lenght - * @return CRC32 - */ -static uint32_t crc32c_sw(uint32_t crci, const void *buf, size_t len) -{ - const unsigned char *next = buf; - uint64_t crc; - - pthread_once(&crc32c_once_sw, crc32c_init_sw); - crc = crci ^ 0xffffffff; - while (len && ((uintptr_t)next & 7) != 0) { - crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); - len--; - } - while (len >= 8) { - crc ^= *(uint64_t *)next; - crc = crc32c_table[7][crc & 0xff] ^ - crc32c_table[6][(crc >> 8) & 0xff] ^ - crc32c_table[5][(crc >> 16) & 0xff] ^ - crc32c_table[4][(crc >> 24) & 0xff] ^ - crc32c_table[3][(crc >> 32) & 0xff] ^ - crc32c_table[2][(crc >> 40) & 0xff] ^ - crc32c_table[1][(crc >> 48) & 0xff] ^ - crc32c_table[0][crc >> 56]; - next += 8; - len -= 8; - } - while (len) { - crc = crc32c_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); - len--; - } - return (uint32_t)crc ^ 0xffffffff; -} - -/** - * @brief Multiply a matrix times a vector over the Galois field of two elements, - GF(2). Each element is a bit in an unsigned integer. mat must have at - least as many entries as the power of two for most significant one bit in - vec. - * @param mat Input matrix - * @param vec Input vector - * @return result - */ -static inline uint32_t gf2_matrix_times(uint32_t *mat, uint32_t vec) -{ - uint32_t sum; - - sum = 0; - while (vec) { - if (vec & 1) - sum ^= *mat; - vec >>= 1; - mat++; - } - return sum; -} - -/** - * @brief Multiply a matrix by itself over GF(2). Both mat and square must have 32 - rows. - * @param square Output pointer - * @param mat Input matrix - */ -static inline void gf2_matrix_square(uint32_t *square, uint32_t *mat) -{ - int n; - - for (n = 0; n < 32; n++) - square[n] = gf2_matrix_times(mat, mat[n]); -} - -/** - * @brief Construct an operator to apply len zeros to a crc. len must be a power of - two. If len is not a power of two, then the result is the same as for the - largest power of two less than len. The result for len == 0 is the same as - for len == 1. A version of this routine could be easily written for any - len, but that is not needed for this application. - * @param even //TODO - * @param len //TODO - */ -static void crc32c_zeros_op(uint32_t *even, size_t len) -{ - int n; - uint32_t row; - uint32_t odd[32]; /* odd-power-of-two zeros operator */ - - /* put operator for one zero bit in odd */ - odd[0] = POLY; /* CRC-32C polynomial */ - row = 1; - for (n = 1; n < 32; n++) { - odd[n] = row; - row <<= 1; - } - - /* put operator for two zero bits in even */ - gf2_matrix_square(even, odd); - - /* put operator for four zero bits in odd */ - gf2_matrix_square(odd, even); - - /* first square will put the operator for one zero byte (eight zero bits), - in even -- next square puts operator for two zero bytes in odd, and so - on, until len has been rotated down to zero */ - do { - gf2_matrix_square(even, odd); - len >>= 1; - if (len == 0) - return; - gf2_matrix_square(odd, even); - len >>= 1; - } while (len); - - /* answer ended up in odd -- copy to even */ - for (n = 0; n < 32; n++) - even[n] = odd[n]; -} - -/** - * @brief Take a length and build four lookup tables for applying the zeros operator - for that length, byte-by-byte on the operand. - * @param zeros //TODO - * @param len //TODO - */ -static void crc32c_zeros(uint32_t zeros[][256], size_t len) -{ - uint32_t n; - uint32_t op[32]; - - crc32c_zeros_op(op, len); - for (n = 0; n < 256; n++) { - zeros[0][n] = gf2_matrix_times(op, n); - zeros[1][n] = gf2_matrix_times(op, n << 8); - zeros[2][n] = gf2_matrix_times(op, n << 16); - zeros[3][n] = gf2_matrix_times(op, n << 24); - } -} - -/** - * @brief Apply the zeros operator table to crc. - * @param zeros //TODO - * @param crc //TODO - * @return //TODO - */ -static inline uint32_t crc32c_shift(uint32_t zeros[][256], uint32_t crc) -{ - return zeros[0][crc & 0xff] ^ zeros[1][(crc >> 8) & 0xff] ^ - zeros[2][(crc >> 16) & 0xff] ^ zeros[3][crc >> 24]; -} - -/* Block sizes for three-way parallel crc computation. LONG and SHORT must - both be powers of two. The associated string constants must be set - accordingly, for use in constructing the assembler instructions. */ -#define LONG 8192 -#define LONGx1 "8192" -#define LONGx2 "16384" -#define SHORT 256 -#define SHORTx1 "256" -#define SHORTx2 "512" - -/* Tables for hardware crc that shift a crc by LONG and SHORT zeros. */ -static pthread_once_t crc32c_once_hw = PTHREAD_ONCE_INIT; -static uint32_t crc32c_long[4][256]; -static uint32_t crc32c_short[4][256]; - -/** - * @brief Initialize tables for shifting crcs. - */ -static void crc32c_init_hw(void) -{ - crc32c_zeros(crc32c_long, LONG); - crc32c_zeros(crc32c_short, SHORT); -} - -/** - * @brief Compute CRC-32C using the Intel hardware instruction. - * @param crc //TODO - * @param buf //TODO - * @param len //TODO - * @return //TODO - */ -static uint32_t crc32c_hw(uint32_t crc, const void *buf, size_t len) -{ - const unsigned char *next = buf; - const unsigned char *end; - uint64_t crc0, crc1, crc2; /* need to be 64 bits for crc32q */ - - /* populate shift tables the first time through */ - pthread_once(&crc32c_once_hw, crc32c_init_hw); - - /* pre-process the crc */ - crc0 = crc ^ 0xffffffff; - - /* compute the crc for up to seven leading bytes to bring the data pointer - to an eight-byte boundary */ - while (len && ((uintptr_t)next & 7) != 0) { - __asm__("crc32b\t" "(%1), %0" - : "=r"(crc0) - : "r"(next), "0"(crc0)); - next++; - len--; - } - - /* compute the crc on sets of LONG*3 bytes, executing three independent crc - instructions, each on LONG bytes -- this is optimized for the Nehalem, - Westmere, Sandy Bridge, and Ivy Bridge architectures, which have a - throughput of one crc per cycle, but a latency of three cycles */ - while (len >= LONG*3) { - crc1 = 0; - crc2 = 0; - end = next + LONG; - do { - __asm__("crc32q\t" "(%3), %0\n\t" - "crc32q\t" LONGx1 "(%3), %1\n\t" - "crc32q\t" LONGx2 "(%3), %2" - : "=r"(crc0), "=r"(crc1), "=r"(crc2) - : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2)); - next += 8; - } while (next < end); - crc0 = crc32c_shift(crc32c_long, crc0) ^ crc1; - crc0 = crc32c_shift(crc32c_long, crc0) ^ crc2; - next += LONG*2; - len -= LONG*3; - } - - /* do the same thing, but now on SHORT*3 blocks for the remaining data less - than a LONG*3 block */ - while (len >= SHORT*3) { - crc1 = 0; - crc2 = 0; - end = next + SHORT; - do { - __asm__("crc32q\t" "(%3), %0\n\t" - "crc32q\t" SHORTx1 "(%3), %1\n\t" - "crc32q\t" SHORTx2 "(%3), %2" - : "=r"(crc0), "=r"(crc1), "=r"(crc2) - : "r"(next), "0"(crc0), "1"(crc1), "2"(crc2)); - next += 8; - } while (next < end); - crc0 = crc32c_shift(crc32c_short, crc0) ^ crc1; - crc0 = crc32c_shift(crc32c_short, crc0) ^ crc2; - next += SHORT*2; - len -= SHORT*3; - } - - /* compute the crc on the remaining eight-byte units less than a SHORT*3 - block */ - end = next + (len - (len & 7)); - while (next < end) { - __asm__("crc32q\t" "(%1), %0" - : "=r"(crc0) - : "r"(next), "0"(crc0)); - next += 8; - } - len &= 7; - - /* compute the crc for up to seven trailing bytes */ - while (len) { - __asm__("crc32b\t" "(%1), %0" - : "=r"(crc0) - : "r"(next), "0"(crc0)); - next++; - len--; - } - - /* return a post-processed crc */ - return (uint32_t)crc0 ^ 0xffffffff; -} - -/* Check for SSE 4.2. SSE 4.2 was first supported in Nehalem processors - introduced in November, 2008. This does not check for the existence of the - cpuid instruction itself, which was introduced on the 486SL in 1992, so this - will fail on earlier x86 processors. cpuid works on all Pentium and later - processors. */ -#ifdef __x86_64__ -#define SSE42(have) \ - do { \ - uint32_t eax, ecx; \ - eax = 1; \ - __asm__("cpuid" \ - : "=c"(ecx) \ - : "a"(eax) \ - : "%ebx", "%edx"); \ - (have) = (ecx >> 20) & 1; \ - } while (0) -#endif - -/** - * @brief Compute a CRC-32C. If the crc32 instruction is available, use the hardware - version. Otherwise, use the software version. - * @param crc //TODO - * @param buf //TODO - * @param len //TODO - * @return //TODO - */ -uint32_t crc32c(uint32_t crc, const void *buf, size_t len) -{ - int sse42; -#ifdef __x86_64__ - SSE42(sse42); - return sse42 ? crc32c_hw(crc, buf, len) : crc32c_sw(crc, buf, len); -#else - return crc32c_sw(crc, buf, len); -#endif -} - - -#define SIZE (262144*3) -#define CHUNK SIZE - -/** - * @brief //TODO - * @param data //TODO - * @param len //TODO - * @return //TODO - */ -uint32_t calc_crc32c (char *data, ssize_t len) { - - uint32_t crc = 0; - size_t off, n; - off = 0; - - do { - n = (size_t)len - off; - if (n > CHUNK) n = CHUNK; - crc = crc32c(crc, data + off, n); - off += n; - } while (off < (size_t)len); - - return crc; -} - diff --git a/tests/data/header-files-test/deploy.sh b/tests/data/header-files-test/deploy.sh deleted file mode 100644 index 44f22a65..00000000 --- a/tests/data/header-files-test/deploy.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024 HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 -# -# This script deploys the application to the target environment. -# It handles building, testing, and deploying in a single step. - -set -euo pipefail - -# Configuration -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" -readonly BUILD_DIR="${PROJECT_ROOT}/build" -readonly DEPLOY_ENV="${1:-staging}" -readonly VERSION="${2:-$(git describe --tags --always)}" -readonly TIMESTAMP="$(date -u +%Y%m%d%H%M%S)" - -# Colors for output -readonly RED='\033[0;31m' -readonly GREEN='\033[0;32m' -readonly YELLOW='\033[1;33m' -readonly NC='\033[0m' - -log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" >&2 -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $*" >&2 -} - -check_prerequisites() { - local missing=() - - for cmd in docker kubectl jq; do - if ! command -v "$cmd" &> /dev/null; then - missing+=("$cmd") - fi - done - - if [ ${#missing[@]} -gt 0 ]; then - log_error "Missing required tools: ${missing[*]}" - exit 1 - fi - - log_info "All prerequisites met" -} - -build_image() { - local image_tag="myapp:${VERSION}" - - log_info "Building Docker image: ${image_tag}" - - docker build \ - --build-arg VERSION="${VERSION}" \ - --build-arg BUILD_DATE="${TIMESTAMP}" \ - --tag "${image_tag}" \ - --file "${PROJECT_ROOT}/Dockerfile" \ - "${PROJECT_ROOT}" - - log_info "Image built successfully: ${image_tag}" - echo "${image_tag}" -} - -run_tests() { - log_info "Running test suite..." - - if ! docker run --rm "myapp:${VERSION}" test; then - log_error "Tests failed" - exit 1 - fi - - log_info "All tests passed" -} - -deploy() { - local env="$1" - local image_tag="$2" - - log_info "Deploying ${image_tag} to ${env}" - - kubectl set image "deployment/myapp" \ - "myapp=${image_tag}" \ - --namespace="${env}" \ - --record - - kubectl rollout status "deployment/myapp" \ - --namespace="${env}" \ - --timeout=300s - - log_info "Deployment to ${env} completed successfully" -} - -main() { - log_info "Starting deployment pipeline" - log_info "Environment: ${DEPLOY_ENV}" - log_info "Version: ${VERSION}" - - check_prerequisites - - local image_tag - image_tag=$(build_image) - - run_tests - - deploy "${DEPLOY_ENV}" "${image_tag}" - - log_info "Pipeline completed successfully" -} - -main "$@" \ No newline at end of file diff --git a/tests/data/header-files-test/handler.go b/tests/data/header-files-test/handler.go deleted file mode 100644 index 2c65d8ef..00000000 --- a/tests/data/header-files-test/handler.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2024 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package http - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "sync" - "time" -) - -// Handler responds to an HTTP request. -type Handler interface { - ServeHTTP(w http.ResponseWriter, r *http.Request) -} - -// Router is an HTTP request multiplexer that matches the URL -// of each incoming request against a list of registered patterns. -type Router struct { - mu sync.RWMutex - routes map[string]Handler - notFound Handler -} - -// NewRouter creates a new Router instance. -func NewRouter() *Router { - return &Router{ - routes: make(map[string]Handler), - } -} - -// Handle registers the handler for the given pattern. -func (r *Router) Handle(pattern string, handler Handler) { - r.mu.Lock() - defer r.mu.Unlock() - r.routes[pattern] = handler -} - -// HandleFunc registers the handler function for the given pattern. -func (r *Router) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) { - r.Handle(pattern, http.HandlerFunc(handler)) -} - -// ServeHTTP dispatches the request to the handler whose -// pattern most closely matches the request URL. -func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { - r.mu.RLock() - handler, ok := r.routes[req.URL.Path] - r.mu.RUnlock() - - if !ok { - if r.notFound != nil { - r.notFound.ServeHTTP(w, req) - return - } - http.NotFound(w, req) - return - } - - handler.ServeHTTP(w, req) -} - -// JSONResponse writes a JSON response with the given status code. -func JSONResponse(w http.ResponseWriter, statusCode int, data interface{}) error { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(statusCode) - return json.NewEncoder(w).Encode(data) -} - -// ReadJSON reads a JSON request body into the given destination. -func ReadJSON(r *http.Request, dst interface{}) error { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - return fmt.Errorf("reading body: %w", err) - } - return json.Unmarshal(body, dst) -} \ No newline at end of file diff --git a/tests/data/header-files-test/logger.rb b/tests/data/header-files-test/logger.rb deleted file mode 100644 index ad423ac3..00000000 --- a/tests/data/header-files-test/logger.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -# Copyright (c) 2024 Rails Contributors -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - -require 'logger' -require 'fileutils' -require 'singleton' - -module ActiveSupport - class TaggedLogging - include Singleton - - SEVERITIES = %i[debug info warn error fatal unknown].freeze - DEFAULT_FORMAT = "%s [%s] %s -- %s: %s\n" - - attr_reader :logger, :tags - - def initialize - @logger = ::Logger.new($stdout) - @logger.formatter = method(:default_formatter) - @tags = [] - @mutex = Mutex.new - end - - def tagged(*new_tags, &block) - @mutex.synchronize do - @tags.concat(new_tags.flatten) - result = block.call(self) - @tags.pop(new_tags.flatten.size) - result - end - end - - SEVERITIES.each do |severity| - define_method(severity) do |message = nil, &block| - message = block.call if block - return if message.nil? - - formatted_tags = @tags.map { |t| "[#{t}]" }.join(" ") - @logger.send(severity, "#{formatted_tags} #{message}".strip) - end - end - - def silence(temporary_level = ::Logger::ERROR, &block) - old_level = @logger.level - @logger.level = temporary_level - yield self - ensure - @logger.level = old_level - end - - private - - def default_formatter(severity, datetime, progname, msg) - format(DEFAULT_FORMAT, - severity[0], - datetime.strftime("%Y-%m-%dT%H:%M:%S.%6N"), - $$, - progname, - msg) - end - end -end \ No newline at end of file diff --git a/tests/data/header-files-test/multiline_imports.py b/tests/data/header-files-test/multiline_imports.py deleted file mode 100644 index 8e22d972..00000000 --- a/tests/data/header-files-test/multiline_imports.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -SPDX-License-Identifier: MIT - - Copyright (c) 2021, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -""" - -import argparse -import os -import sys -import traceback -from dataclasses import asdict -from pathlib import Path -from typing import List - -import pypac - -from scanoss.cryptography import Cryptography, create_cryptography_config_from_args -from scanoss.delta import Delta -from scanoss.export.dependency_track import DependencyTrackExporter -from scanoss.scanners.container_scanner import ( - DEFAULT_SYFT_COMMAND, - DEFAULT_SYFT_TIMEOUT, - ContainerScanner, - create_container_scanner_config_from_args, -) -from scanoss.scanners.folder_hasher import ( - FolderHasher, - create_folder_hasher_config_from_args, -) -from scanoss.scanossgrpc import ( - ScanossGrpc, - ScanossGrpcError, - create_grpc_config_from_args, -) - -from .components import Components -from .constants import ( - DEFAULT_API_TIMEOUT, - DEFAULT_COPYLEFT_LICENSE_SOURCES, - DEFAULT_HFH_DEPTH, - DEFAULT_HFH_MIN_ACCEPTED_SCORE, - DEFAULT_HFH_RANK_THRESHOLD, - DEFAULT_HFH_RECURSIVE_THRESHOLD, - DEFAULT_POST_SIZE, - DEFAULT_RETRY, - DEFAULT_TIMEOUT, - MIN_TIMEOUT, - PYTHON_MAJOR_VERSION, - VALID_LICENSE_SOURCES, -) -from .csvoutput import CsvOutput -from .cyclonedx import CycloneDx -from .filecount import FileCount -from .gitlabqualityreport import GitLabQualityReport -from .inspection.policy_check.dependency_track.project_violation import ( - DependencyTrackProjectViolationPolicyCheck, -) -from .inspection.policy_check.scanoss.copyleft import Copyleft -from .inspection.policy_check.scanoss.undeclared_component import UndeclaredComponent -from .inspection.summary.component_summary import ComponentSummary -from .inspection.summary.license_summary import LicenseSummary -from .inspection.summary.match_summary import MatchSummary -from .results import Results -from .scancodedeps import ScancodeDeps -from .scanner import FAST_WINNOWING, Scanner -from .scanners.scanner_config import create_scanner_config_from_args -from .scanners.scanner_hfh import ScannerHFH -from .scanoss_settings import ScanossSettings, ScanossSettingsError -from .scantype import ScanType -from .spdxlite import SpdxLite -from .threadeddependencies import SCOPE -from .utils.file import validate_json_file - -HEADER_PARTS_COUNT = 2 - - -def print_stderr(*args, **kwargs): - """ - Print the given message to STDERR - """ - print(*args, file=sys.stderr, **kwargs) \ No newline at end of file diff --git a/tests/data/header-files-test/parser.pl b/tests/data/header-files-test/parser.pl deleted file mode 100644 index 097090e1..00000000 --- a/tests/data/header-files-test/parser.pl +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/perl -# Copyright (c) 2024 The Perl Foundation -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the Artistic License 2.0. -# -# See https://opensource.org/licenses/Artistic-2.0 -# for the full license text. - -use strict; -use warnings; -use Getopt::Long; -use File::Basename; -use Carp qw(croak); - -my $VERSION = '1.0.0'; - -sub new { - my ($class, %args) = @_; - my $self = bless { - delimiter => $args{delimiter} || ',', - quote_char => $args{quote_char} || '"', - escape => $args{escape} || '\\', - headers => $args{headers} || [], - strict => $args{strict} // 1, - line_num => 0, - }, $class; - return $self; -} - -sub parse_file { - my ($self, $filename) = @_; - croak "Filename required" unless defined $filename; - - open my $fh, '<:encoding(UTF-8)', $filename - or croak "Cannot open '$filename': $!"; - - my @records; - my $header_line = <$fh>; - chomp $header_line; - $self->{headers} = $self->_split_line($header_line); - $self->{line_num} = 1; - - while (my $line = <$fh>) { - chomp $line; - $self->{line_num}++; - next if $line =~ /^\s*$/; - next if $line =~ /^\s*#/; - - my $fields = $self->_split_line($line); - if ($self->{strict} && scalar @$fields != scalar @{$self->{headers}}) { - croak sprintf( - "Field count mismatch at line %d: expected %d, got %d", - $self->{line_num}, - scalar @{$self->{headers}}, - scalar @$fields - ); - } - - my %record; - for my $i (0 .. $#{$self->{headers}}) { - $record{$self->{headers}[$i]} = $fields->[$i] // ''; - } - push @records, \%record; - } - - close $fh; - return \@records; -} - -sub _split_line { - my ($self, $line) = @_; - my @fields; - my $field = ''; - my $in_quotes = 0; - - for my $char (split //, $line) { - if ($char eq $self->{quote_char} && !$in_quotes) { - $in_quotes = 1; - } elsif ($char eq $self->{quote_char} && $in_quotes) { - $in_quotes = 0; - } elsif ($char eq $self->{delimiter} && !$in_quotes) { - push @fields, $field; - $field = ''; - } else { - $field .= $char; - } - } - push @fields, $field; - - return \@fields; -} - -1; \ No newline at end of file diff --git a/tests/data/header-files-test/results.py b/tests/data/header-files-test/results.py deleted file mode 100644 index 5cbff282..00000000 --- a/tests/data/header-files-test/results.py +++ /dev/null @@ -1,275 +0,0 @@ -""" -SPDX-License-Identifier: MIT - - Copyright (c) 2024, SCANOSS - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -""" - -import json -from typing import Any, Dict, List - -from scanoss.utils.abstract_presenter import AbstractPresenter - -from .scanossbase import ScanossBase - -MATCH_TYPES = ['file', 'snippet'] -STATUSES = ['pending', 'identified'] - - -AVAILABLE_FILTER_VALUES = { - 'match_type': [e for e in MATCH_TYPES], - 'status': [e for e in STATUSES], -} - - -ARG_TO_FILTER_MAP = { - 'match_type': 'id', - 'status': 'status', -} - -PENDING_IDENTIFICATION_FILTERS = { - 'match_type': ['file', 'snippet'], - 'status': ['pending'], -} - - -class ResultsPresenter(AbstractPresenter): - """ - SCANOSS Results presenter class - Handles the presentation of the scan results - """ - - def __init__(self, results_instance, **kwargs): - super().__init__(**kwargs) - self.results = results_instance - - def _format_json_output(self) -> str: - """ - Format the output data into a JSON object - """ - - formatted_data = [] - for item in self.results.data: - formatted_data.append( - { - 'file': item.get('filename'), - 'status': item.get('status', 'N/A'), - 'match_type': item['id'], - 'matched': item.get('matched', 'N/A'), - 'purl': (item.get('purl')[0] if item.get('purl') else 'N/A'), - 'license': (item.get('licenses')[0].get('name', 'N/A') if item.get('licenses') else 'N/A'), - } - ) - try: - return json.dumps({'results': formatted_data, 'total': len(formatted_data)}, indent=2) - except Exception as e: - self.base.print_stderr(f'ERROR: Problem formatting JSON output: {e}') - return '' - - def _format_cyclonedx_output(self) -> str: - raise NotImplementedError('CycloneDX output is not implemented') - - def _format_spdxlite_output(self) -> str: - raise NotImplementedError('SPDXlite output is not implemented') - - def _format_csv_output(self) -> str: - raise NotImplementedError('CSV output is not implemented') - - def _format_raw_output(self) -> str: - raise NotImplementedError('Raw output is not implemented') - - def _format_plain_output(self) -> str: - """Format the output data into a plain text string - - Returns: - str: The formatted output data - """ - if not self.results.data: - msg = 'No results to present' - return msg - - formatted = '' - for item in self.results.data: - formatted += f'{self._format_plain_output_item(item)}\n' - return formatted - - @staticmethod - def _format_plain_output_item(item): - purls = item.get('purl', []) - licenses = item.get('licenses', []) - - return ( - f'File: {item.get("filename")}\n' - f'Match type: {item.get("id")}\n' - f'Status: {item.get("status", "N/A")}\n' - f'Matched: {item.get("matched", "N/A")}\n' - f'Purl: {purls[0] if purls else "N/A"}\n' - f'License: {licenses[0].get("name", "N/A") if licenses else "N/A"}\n' - ) - - -class Results: - """ - SCANOSS Results class \n - Handles the parsing and filtering of the scan results - """ - - def __init__( # noqa: PLR0913 - self, - debug: bool = False, - trace: bool = False, - quiet: bool = False, - filepath: str = None, - match_type: str = None, - status: str = None, - output_file: str = None, - output_format: str = None, - ): - """Initialise the Results class - - Args: - debug (bool, optional): Debug. Defaults to False. - trace (bool, optional): Trace. Defaults to False. - quiet (bool, optional): Quiet. Defaults to False. - filepath (str, optional): Path to the scan results file. Defaults to None. - match_type (str, optional): Comma separated match type filters. Defaults to None. - status (str, optional): Comma separated status filters. Defaults to None. - output_file (str, optional): Path to the output file. Defaults to None. - output_format (str, optional): Output format. Defaults to None. - """ - - self.base = ScanossBase(debug, trace, quiet) - self.data = self._load_and_transform(filepath) - self.filters = self._load_filters(match_type=match_type, status=status) - self.presenter = ResultsPresenter( - self, - debug=debug, - trace=trace, - quiet=quiet, - output_file=output_file, - output_format=output_format, - ) - - def load_file(self, file: str) -> Dict[str, Any]: - """Load the JSON file - - Args: - file (str): Path to the JSON file - - Returns: - Dict[str, Any]: The parsed JSON data - """ - with open(file, 'r') as jsonfile: - try: - return json.load(jsonfile) - except Exception as e: - self.base.print_stderr(f'ERROR: Problem parsing input JSON: {e}') - - def _load_and_transform(self, file: str) -> List[Dict[str, Any]]: - """ - Load the file and transform the data into a list of dictionaries with the filename and the file data - """ - - raw_data = self.load_file(file) - return self._transform_data(raw_data) - - @staticmethod - def _transform_data(data: dict) -> list: - """Transform the data into a list of dictionaries with the filename and the file data - - Args: - data (dict): The raw data - - Returns: - list: The transformed data - """ - result = [] - for filename, file_data in data.items(): - if file_data: - file_obj = {'filename': filename} - file_obj.update(file_data[0]) - result.append(file_obj) - return result - - def _load_filters(self, **kwargs): - """Extract and parse the filters - - Returns: - dict: Parsed filters - """ - filters = {} - - for key, value in kwargs.items(): - if value: - filters[key] = self._extract_comma_separated_values(value) - - return filters - - @staticmethod - def _extract_comma_separated_values(values: str): - return [value.strip() for value in values.split(',')] - - def apply_filters(self): - """Apply the filters to the data""" - filtered_data = [] - for item in self.data: - if self._item_matches_filters(item): - filtered_data.append(item) - self.data = filtered_data - - return self - - def _item_matches_filters(self, item): - for filter_key, filter_values in self.filters.items(): - if not filter_values: - continue - - self._validate_filter_values(filter_key, filter_values) - - item_value = item.get(ARG_TO_FILTER_MAP[filter_key]) - if isinstance(filter_values, list): - if item_value not in filter_values: - return False - elif item_value != filter_values: - return False - return True - - @staticmethod - def _validate_filter_values(filter_key: str, filter_value: List[str]): - if any(value not in AVAILABLE_FILTER_VALUES.get(filter_key, []) for value in filter_value): - valid_values = ', '.join(AVAILABLE_FILTER_VALUES.get(filter_key, [])) - raise ValueError( - f"ERROR: Invalid filter value '{filter_value}' for filter '{filter_key}'. " - f'Valid values are: {valid_values}' - ) - - def get_pending_identifications(self): - """Get files with 'pending' status and 'file' or 'snippet' match type""" - self.filters = PENDING_IDENTIFICATION_FILTERS - self.apply_filters() - - return self - - def has_results(self): - return bool(self.data) - - def present(self, output_format: str = None, output_file: str = None): - """Present the results in the selected format""" - self.presenter.present(output_format=output_format, output_file=output_file) diff --git a/tests/data/header-files-test/server.ex b/tests/data/header-files-test/server.ex deleted file mode 100644 index c75a2e40..00000000 --- a/tests/data/header-files-test/server.ex +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) 2024 Plataformatec -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -defmodule MyApp.Server do - use GenServer - require Logger - alias MyApp.Config - import MyApp.Utils, only: [format_timestamp: 1] - - @moduledoc """ - A GenServer that manages TCP connections and handles - incoming requests from clients. - """ - - @default_port 4000 - @max_connections 100 - @timeout 30_000 - - defstruct [:socket, :port, :connections, :started_at] - - def start_link(opts \\ []) do - port = Keyword.get(opts, :port, @default_port) - GenServer.start_link(__MODULE__, port, name: __MODULE__) - end - - @impl true - def init(port) do - Logger.info("Starting server on port #{port}") - - case :gen_tcp.listen(port, [:binary, active: false, reuseaddr: true]) do - {:ok, socket} -> - state = %__MODULE__{ - socket: socket, - port: port, - connections: %{}, - started_at: DateTime.utc_now() - } - - {:ok, state, {:continue, :accept}} - - {:error, reason} -> - Logger.error("Failed to listen on port #{port}: #{inspect(reason)}") - {:stop, reason} - end - end - - @impl true - def handle_continue(:accept, state) do - case :gen_tcp.accept(state.socket, @timeout) do - {:ok, client} -> - Logger.debug("New connection accepted") - {:ok, pid} = Task.start(fn -> handle_client(client) end) - :gen_tcp.controlling_process(client, pid) - - connections = Map.put(state.connections, pid, client) - {:noreply, %{state | connections: connections}, {:continue, :accept}} - - {:error, :timeout} -> - {:noreply, state, {:continue, :accept}} - - {:error, reason} -> - Logger.error("Accept error: #{inspect(reason)}") - {:stop, reason, state} - end - end - - @impl true - def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do - connections = Map.delete(state.connections, pid) - {:noreply, %{state | connections: connections}} - end - - defp handle_client(socket) do - case :gen_tcp.recv(socket, 0) do - {:ok, data} -> - response = process_request(data) - :gen_tcp.send(socket, response) - handle_client(socket) - - {:error, :closed} -> - Logger.debug("Client disconnected") - :ok - end - end - - defp process_request(data) do - "HTTP/1.1 200 OK\r\nContent-Length: #{byte_size(data)}\r\n\r\n#{data}" - end -end \ No newline at end of file diff --git a/tests/data/header-files-test/server.js b/tests/data/header-files-test/server.js deleted file mode 100644 index 94bc11f1..00000000 --- a/tests/data/header-files-test/server.js +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2023 Express Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -'use strict'; - -const http = require('http'); -const path = require('path'); -const fs = require('fs'); - -const DEFAULT_PORT = 3000; -const DEFAULT_HOST = '127.0.0.1'; - -class Server { - constructor(options = {}) { - this.port = options.port || DEFAULT_PORT; - this.host = options.host || DEFAULT_HOST; - this.routes = new Map(); - this.middleware = []; - } - - use(fn) { - if (typeof fn !== 'function') { - throw new TypeError('Middleware must be a function'); - } - this.middleware.push(fn); - return this; - } - - get(path, handler) { - this.routes.set(`GET:${path}`, handler); - return this; - } - - post(path, handler) { - this.routes.set(`POST:${path}`, handler); - return this; - } - - listen(callback) { - this.server = http.createServer((req, res) => { - this._handleRequest(req, res); - }); - - this.server.listen(this.port, this.host, () => { - if (callback) callback(this.port, this.host); - }); - - return this; - } - - _handleRequest(req, res) { - const key = `${req.method}:${req.url}`; - const handler = this.routes.get(key); - - if (handler) { - handler(req, res); - } else { - res.statusCode = 404; - res.end('Not Found'); - } - } - - close() { - if (this.server) { - this.server.close(); - } - } -} - -module.exports = Server; \ No newline at end of file diff --git a/tests/data/header-files-test/widget.dart b/tests/data/header-files-test/widget.dart deleted file mode 100644 index 6a5fb0bf..00000000 --- a/tests/data/header-files-test/widget.dart +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter/foundation.dart'; -import 'package:flutter/rendering.dart'; -import 'package:flutter/widgets.dart'; - -/// A material design card widget. -/// -/// A card is a sheet of material used to represent some related information, -/// for example an album, a geographical location, a meal, contact details, etc. -class MaterialCard extends StatelessWidget { - const MaterialCard({ - super.key, - this.color, - this.elevation = 1.0, - this.shape, - this.borderOnForeground = true, - this.margin, - this.clipBehavior = Clip.none, - this.child, - }); - - final Color? color; - final double elevation; - final ShapeBorder? shape; - final bool borderOnForeground; - final EdgeInsetsGeometry? margin; - final Clip clipBehavior; - final Widget? child; - - @override - Widget build(BuildContext context) { - final CardThemeData cardTheme = CardTheme.of(context); - final CardThemeData defaults = _CardDefaults(context); - - return Semantics( - container: true, - child: Container( - margin: margin ?? cardTheme.margin ?? defaults.margin, - child: Material( - type: MaterialType.card, - color: color ?? cardTheme.color ?? defaults.color, - elevation: elevation, - shape: shape ?? cardTheme.shape ?? defaults.shape, - borderOnForeground: borderOnForeground, - clipBehavior: clipBehavior, - child: Semantics( - explicitChildNodes: true, - child: child, - ), - ), - ), - ); - } -} - -class _CardDefaults extends CardThemeData { - _CardDefaults(this.context); - - final BuildContext context; - - @override - Color? get color => Theme.of(context).cardColor; - - @override - double? get elevation => 1.0; - - @override - EdgeInsetsGeometry? get margin => const EdgeInsets.all(4.0); - - @override - ShapeBorder? get shape => const RoundedRectangleBorder( - borderRadius: BorderRadius.all(Radius.circular(12.0)), - ); -} \ No newline at end of file diff --git a/tests/data/src.tar.gz b/tests/data/src.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..682c9262d953ce5551cbe481257d0fc89c12b268 GIT binary patch literal 29169 zcmV(>K-j+@iwFR=hOub?1MGckW7{^e=>1y#3RFpIDwXL+9_^81H;OGM+VyKCx!qpJ zN2MjoW+Ra*N!jtbzQ6s>0{{V%@*}VAo)hghmhc!127|$1Fc>89VCnyS=Fjr-@(<6R z3Htk-|1PgS<-hVHR-Uaqd-`Pg$;y)zvAnYK!_z1KM?Cx7h~a0Jq+SfDc&D@ErgBhh zG`b;Q_Cx>u5BVf8{%2k|9Q(0*8hxHNfF94EKAk)MPnVxO;qhO6zWl?JXE6RxeprR^ zU;aGk*q{I9KjG|dM6hR1yzSwDRw)VO$F$%`MTd7RF!8`BN7iVcYtyF@^G>TKN} zr&ajshtO~ko-X~KL?I@OCX^mT_;2W^OIRDfCut1j68d@>4)AXpO#Eu4Rzc+6@I=6V zPNQ&`h%^#_;+tOY0Mh=L`AOLM18WjevCRj#SoPBcpqR+4W_OlIRWn2!DTkWCbuc8 zH!+q=AbH|VkC>Xrk`|S3!YJLQp;Rf{y*>Xzf*-`bht)k1`WL#}fTI@jg`iY^iv zCssJrBoIdy5Zqc_gHg>AvhHN&?nHxkPOb9I&>ta7VbWnZmWi>;=pRXN$6{SUvT)u; zEau=1S!oc3_v`>doZAVkU0XljV(4 zst^!cVOs)?hvIMD{XInmae-vOd5Nh~{9$sCo*AqK^E90$B;_TIO(63~%e6LXX$aH0 z7<#Ff$<3?ln(*T|qDg+?zVW>wPJKW-b*h^v7K%T_;vxwD z$rM&SvDw*Ne6qSJW(f)y6pC#SZ(}Y?Ivr|~AQ~&;v9ZH&HfylUSa~9Lz4)DIlD-24 zeg@?RRhbocMTiyHdz^A9yVw#4>c_9>F^;g05#qp)CqY8p#4QrG4j9u8<1z|; z{5GSFk=Z%b;B*#GVQHuX9MJQ}GZO1C785@n1m0K}D`QfJ8Vq<$`gKabLFo@r-8;e2 z3sL|yihaSbu^5cKV8Xi^%)E%B)F%WOJakHvPqar=ra&Kt1h{y}14H0A z?7V;0OA1I1L1C+v%!VaMt5X89g z6=D^)#Jq`;+un2L-h?{{ou5M2LQ+uUcTmg&glB^@9v6d1+@wt@EPUdtIaMSD)Oq6a z3;-Z7X`zuzMfBL8z|MP!obE{!i7%0ZB(9nXNNcIsGYnwimzUmnHVFfFfItRku0Ncq z*@x5bC2q~NP{0JxqkOUQRBZVJv>_~G+DRPvBSBVzGcO(zgMiR5MT3ClF>L#Z-v0`wb#Y4n`so0` zHZAUvV`oVb57obsHy$sZ04LS)_k|B~0si8AOr|~*LZoo=N4STE{w6*fDU!z0(kV!v z*@+89mXd)NMoF^dhcJ--(#beFS(=FnE^B;%N9Fhbl3-k**?~fUm z2P|-W@ghlwpo*St*h#QR6w*M|wbN!HbYp0zO%Vfcn6HdGQcOc4s8z-Zx|`j0O-!Tl zWf)CR;sKtqj~&M`2N@T|pxe8M@dHq)f!5++f9F@R{Qk$)(eo!KKm14ldzly)f6Tn$ z1yoz~(nW6(?Le6*1F{e-+)o4pAyMr|;lS^wlK1&nasgZSgWj7?tGU(R-`i~U+k5Ta znyNViQaoAB>ZFLk(T~SRtIwVvugNL(SWOq$@JJI(su9&VDPq919L&aY-(etm9tA_y zOb`Yspyps&t#U<@Mxz2dbtqOB@75}mfvq?pbYpo19tmKIQVsAL#))qe>3SDLM45z^>0xtp>s`JtS%296XU#AY0E@$z4H{}{sa3$1 z{H|Po&@g*~+T_YJfnoxdbI_1z!9@KfG?ZYy)R{6>!SRx0hPB5*ngSAjI1HdNngvdw zebns`fq)HK0}mBRwLoSIBWp~gq9%zyIYBcS+1+8yBV`!K?^1_Rj8Bj(foOupGeMaF z!txwecT|COiIdq#Y@tyGzKmEOmpNmgk&%PHL5ZgV(tH3i2^L;~i*+W5@-h$PkeL*K z_(3_x0_!or1|2l`7YbHrKuIeuG7FR76i*TcKv`JvVa`%fs?cJu$`Ipx4`XPJwpi3D z`2qjzyd`I|o~AVhXU>I!B;Dn+DLmnuc%Zetm$K0VX*vLvRp22Y398rLI6}b`;7%HM zjTf-p9{Z$?lGJ;>?X1>7O1AJEOXiT&CCv`EcVQ+8q`Y)-x03R9>dZR_aY*~hvG3uL z7wpAo>mLGQh!e}yDN1dH0WMQ?oPcnH5$v9!QQU}go4h#~n4%QhYlpBEksUU&prp*= z&YLn4wst@&Q$WJz5EWD{6i-L1eFoVl8fN11F(lWsf&jdr&B2-?MIU^lwcF$#tYL6Z zFrp;X9BLNlVqFv=&|bzTSaM@y-BcqW|NHMtSckB<1CPM=EJzc7JYx1xBmXK&&xpRb zWlKOaSLOs7+9#{nR2*F-=5WYj`?D13K8?t}e)DkUM}Sr!gZd28AU`Jg+bfmAuL6-h^#y+WIHrJ5;g6CS$Z&<&*BM!ip;OmvCAmmu(%Se01L}w zacpQAxf8b#s=2IHwfi)-;Ibm%EVPW!^I*lE2Zjv7hG-h-&MGJ`UT8Si^ZTH>jC~+` z1&x(NrZ}0I{LpA^Z9dFVG>dJZ24#vcA+t`+8Zbv>ijhKz?8ws@E9!Gb;4%gI?2}ns zw_K|$f~bBiF0#HL0Rzlr9yu=2M#iYKtucudvD zgW$+s4Oer8Zm>P59J+7j_|b_V-@XwCujw@AODBLTpY@ z?>vz^gG8z?Qep9uB$7u#Y?Wr4goieCrCB9Df$;`|D8@56JWoTM@T3Za&Oj+{YN}^8 z8{0r+n5_xZA6Ncs)k=B4BC42JwNqC`b>;b!AD?E)+&>sTI|b`QVy+z4f3^ZZwQZb9 zs?AtPND@TECahed4xb|@jn*9v=4baan2)3I6b%DxFP~>&I*Z~oE0vpR?J!6|Ujc?O zMKJYt)tOcfKgd%zHPl2sa3g zJfY8QQ{Jbus3jMn(^;FGh=^IT5j2<3*#dpVn?JE-?p=;y2{Z) z)g!4g#U3W4yNs1@7ioRi@mMhndD!XwFn`#SnY2w-B2LhE(2Fk>LLXx1?DxH7((gM} zqQJ@9v?{93qm^1+JX)^i3}ZqxM0Gu`;#|mSf-)2GuqmirU6#d+C6_H*b8J}ihz6Px zNNAGlI5i(XVRC_1`Q9JQq*TUZ&;T^WDXNPh@mQvWXF|HpZq@*jgdfu#G|E&S5kynq z9AJL6xAeuHf98$-iB{=al)gsf%iY1H-Uso~+s}ex&7^}n%toJjN_X5dVv&g@k8Gl)Gv(_JK8w3hA{9)Br zi)LzKj8%IJwy5|eKLdwRX3>F}fzXi`L)hKJ%%PPJXCbO^+KhyLD;`Ksn2jX5(0Y_{ zI{NeI*-NTlDf>#$AH}}}<8v~wR>khc!CF=OpUuKjJGwUy7weTUF|$XJizVCKVIi>@ z)mTYqAu1c0$6D%gaA1e}iG`=sjuJ5hH6*_oBE!-7r)Wd z2Gd@c&EdZNn{0SUV*k~*Gd}+ZeUJHMHgTtypQnwR@Bgf>E<>h$|L5uQlc!(r|9pv0 zwOXyry#Tb^?%|9!uI~(L6eOT7x|_|t{cg8-u?5`_PcGH{mO9-sK}7?F1{yo~!VB6e z5Sni}0L}&YF?Y-CH71Z3@?}?o@a5RAJ?NgHy zSi99laJ#L}<{S9geAV7*_kKmI-FCaThj6#|JEAELnw?&I^Khrx5eJ8zgMCojn|oUT zc(1*;-GO#myRE&R3++OhX#EU7U?#-Bw*RJMAu#WxKP#i`_)RLY;jA12y+r3PHX4TkX)F;a2|0v9l}hvHW_t%muD{*h>9sojpPM^}REt&41qxHp z$1ocU4ROSmI_PxJK;f8;#0z|XJy%X40);xA+_P<9Q?URfQ*%wl10sX4cz#lBei2xrFeO{HOKo9(EyTN;|q z;xW{7vWk>aL$^BE!sL?$ZVGRPDD?&#s7y=6(kS(iDloc^omsoC)45xit{aAF|NX$9rlLiEW=l!?_2@lcXuRe(+^ zwq|`3YN>RG*^8aIMfy&XQ;^&Xdy!gQXv4q@;6fR1Lxqwh0AVbc$@iB2XjmC7Tv72d zO7{^UNw3=V+c0ZOg{x6;7`d11cQWx|6E?hY1P!n8ps0I<7ruH0&wY+|Kvku5b--boy z&x^@h3OB7V7@#HMuls3%LOg)E_lE~d$WCYZzgF}K9?gbq%^dnCv(tt+iJ~zu(Y80{ z$Q=AZmDZn^`Nu5q(|MW5LsKt3YoOSG>^)dDvfNCKDbG(N+mz+WDKjFjy{Q1rvtp5& zB_fqSowWU1LzB&%CaA==%pkG}2RmOUcbO>k#-p49ibYmbTc*NI(HX3Slm~HDsOmk_D!&?ZI0Fwfs>7$ze8MDXLhPH zU*vXfa$Cep+BouteGn>rs1PQIgQK`1S4|Sot|=3pY^c|>uPQj+I{(SR%eK4g#l3Su zA+8sv+B$c&IE&V~^Ti3WPXE^D4fwk8y}S?&(Yp03D`@CsiZ;I7d27i;IvHAnj))7I z;pMR5Au@`CdSnVc>qq#K4naJrB}KEX1(? zf}ITRny9b2LCNj0@MG??kX(+cYj0iduwTN@UH43C84c*-Bz98mm$K!uWJ;8cB3ko~ zYziKi8#MHw(s+^CSwYSmTBQcAV%ZMabnnsqULB}XsbSJ~Eg=?$$=F|9^v&s>@uf{^ zbqGuoTO&tR9R;IyqY9g=L>e`9f@p=kvS`!$^aQwRWk&c8VZ%%geaclTogaiYRnXnT z6PkImLS6waSez8n%t3wGTY}@FI-58(yIs&Lb;%;HQ&x2HJ6bR^yHZ0pb3M0)JDXxG zD@Owdy|(CA#NAL9FIBg|Eb`uWZr-Q`foDvKeGv%OdQWyy8bcB{Y9z7dEklBXh{BG9 zl&VrT7};`lb#3ho#XJA9&S64u6d1v|<3Qy~#{i)p0jy)aBKq8~K8B+5`dS0&6O6>6 zKgGS5)$cj8>n@O4H#T}8;U7`1i~IShQzHuK&T~aKnT`Y4B-R&d$Juz&9X@M3?NQ7w zcqvlL)dv}?I1+cW#}+vr#(=4d<@z!vvPLVnK`w1laWa)HRfVkLg#p5n2Ah}S!R(&H zM$UaG&?lag-+95SB@ss#2T$$)zFBr2a%Y&w`7hF+voE_{8=IX_p|3;-~ag{pT$M-X9fP*jfOMWDAEUZbf8bW#1;d0Gn$;>GY3(O;s`U;5rCNHOxTJBYAEH}y^eP{J<>1cmBN$Ucaf2T`oE8{}$+ZtM~^nmZ$m#!TUX zPkm?vFQ^mGxx}qhCQO7my%)j}gR=GMk`H1X z&8@F4vu`W(6=Rg~x4{`kj5fX+$+7X3lYI*0R2TGQ2>SK;LkS!q_JT1(!muOFGv7nS zt0pecGgEtGnSDHngNblh3e<)3;?E6qpo}juWQ_lQN^6V(4VF6O8u5~eA*RVo!C}^P zL@IS4gR8V$tjKKf63b#R77W%~M=%_e25+$PQOyKHWe(ZcroOC8_fHOw4G5Fg^^yt$@&KnuYDP%ca4V{vMnm<8 zTFg4gZI^f6tCXP$y&GbPqP}>sNEz_`J$|36L@HR(I@f#EX`Wk9cv|2+IkORGyzYX% z#fwOZ^6afp6reDfO|+d8H>R=)lLRl{1Ld-z&>0A!Kpu$^P{*4~gA8lv`BVT4&gmFB z%--BF`BVX0hp>;JJcBsE8@Du|ZH#3ImALDHNe6F4E_5~tR0p8tY=BMdL1WHv!;2bI zyp*Jl$7?mSH;!gb5_V7?{%W42AX5~4G4@L)Q$0JHLLf+AteuI2%mgYoFNLw-R?GVV zdAUhx;7HRA#k2fr^a*CB*{79Ur$~Z*_CBrLuTv|!S2WG^)Ow%bH;RzyU_K1ww3f{> z=RK^V?-MN^EMaw}Y=99cw9g1iZ{>3oEEiG=FWRJ^}4Ut#6! z)D!6%*+U@L7`l?69xojG81_cqJ_ycf?i(Qk5jHx#tH;=K0~g^ejxx!Sl!*>`DGzTt z#4kRm7sILe! zj~5G+$q_{oAeW<;h3t$K(e+TSNH^8vs-7bOMb_EO}-R&0GZFj&B(!H*M z&ASAKfbM+{EO!fxqOUDrr~saR3u~1M-_^$llW6lx&ucjg_TOp*#xs+Z^jp^cwj^7x? z4)1>Wyj%q=!5QL`J!M^*d*shQ|Fn=;%SxYN$oZ@^{WMA=UPfv8-tsy&}0k6!ib)uloNN`7A9N%0C_Dpi(C4Q*Qu&c9in0UP4@Tm&J>n z_GW9Z+gfZMG&e!Xh++hS^0u+Gba8RvdQ{ns;?pH{^JGcY0RS692EhV7P+d(y2pTS? zV*q4^h&E|{IMap*r8eqz(%q<_jap>eaN-X}0rIv_sP;Sg(xi6EHwI0-1qETLwPxkT~CTGzY zA6SqllnnZ&}Vmda&?L9w987xdEf~@JP+b1#GNAn zIQL?D#TSVA;0Ty4IQN|`z6`KACWnk})>)(Hwc`rx@f7ylX({s<5= zLAUD$J*0-;@+HR6=_w3&L+tPul@E)?N1$n9V<3y`JfKQf7c&%aO=^hQ%JVDWJ`8QBm|V=BtQxsNctdWML$UZHyze`E{`6xzV2zD| zKZ&|x57?z=pM=5-Ikz^9Z0QW7s$J8u?AYclWLa{I&vChWXgZnt0}?09_&A;2tOYK} zuIzlL>^sDRY@8?*_(I8xE{KK=@Ajq%-P}ccQ7qTp)?!=q+@e;$I z{-4w*+O=@g`sd$+5M-TnpxdG=>AtkEst}LmbS7^#LxgDYK@V5O!O%~NDF(HQbgCZ}ff9j{D6>HT*O>Ew7Kh0+=zHyVI~Q|-1oKesxxQLGjo)nrXh zYNB$hiGPgntR*es{GZ#Ym%g5mzM0Irlq`aMk|~(MPb%}Oi5*y)TnWkAgZ6T{Ve)F?f*z+9?!hb(!%Zgzo$PG{r{e>eBJ+jiI3&dr``GX zVYLShA-xpsaNtTux_fTbI3}OWs=^WnAYi+sH%1&fR&AH${)gVo0ht@%Wr*xnn}kB1 zEx$?_e3E3HA?bqn7$_r+pSy1lWu?QAqIpZ>GM`2Bt_IitF$Nxqg$L4(&t3ove0`9D z-K#S!1Y>OR44Nqv%?&Y!REE8dJn#^(oVB}jW}Z9wXt(6)!QNII?ekqb5}qHRCjn5C z5APz1lQs^}Ljav9WfoUawslicK$=Y`UoaFBuk3@f}JYK ze?`5_NH5c2dj>l^k6*c3w|>eJ;VGV@!@u%3U3y=qKQhHf)ee&AuPPPESDM!KSD6Dc z_pCn}M^puWxb#EjMhUGKXp>J|{H8k4-{re;5(w%mR}?E^=p%lR4GKbk*X@In)Wvvt z>tX>`YIk{|Qc*^7DCMF)imM!Lq?E$YPt$Qt85OZ-x=2m@zjaZCwfe-5tK?B3GsFwA zf|L&R(m>tEI3Kcr`QB#|pu>caaJVTylwnrXNQ_51${T)?G@=2!GL|1;r69h{CF&N} z^;aVh1!ovbuhD08r7(U$_ns^pCC>udMHnf_e7=!(Z5gLc?vWQRCTXog9W}=Qjs2XR zU2d!ka}3C64O|zDZ5qplAiXZCSO|j+1>>qsQ0|%R5nl?)dZ~@rltDz6_aaN$)5VaA z8}jC)rngm;h>aVGzJDGJK%>hBEgV+N60RJR?ECj(#qL1?H|cDo^OzA-rkpjP_kJ&Q z7uAUYUjHf#!Kpa}HG>V+<+&UNifA&sgj7r##UJJ`)w{v~(p^Ec^jFI!1awgWGJ2~J z!CX8R&$_-rX$72MUkCdY-_ia2Kx7FAKaVO^mGdO~u7F4YJa&aKkcj1mp@cL4-#l_# z90En`T$@J<*#$0KEOwok$K+vk3z#k%7J4LEpRQ;bBN>fhf1dXFBue37Wl5QjdAyfX z7oTxR1GGd2^4hBGRgHbX^enaDFu5XHc|lH_d4(x!;RgSfWlHP-T47FWM#ykLP;lSMDBuYP7JtmFpVWA#}cJ$j3omLi-n#9C5whJ z&#ZcVbVIZz(2{V>*_4~0HMRIq4`naEx@Gh&N(lNf_ zyB3x9V;ctk4bd+H56I~HY+oiRFdS4uITv<+-R|`__jh-j9u!TzzdwSsX%ra zBBv@kacLWMIgIm_d@O<8X!$7;{2bGNqO?`b?8*bT@yfUhB=o=>3T zLaSA*U)q>_VgN0bQAh`$ifMF;E}~uxFCCEPLEEOf%0L0J(}FRP#>swn|FE;!>e{8= zY`^JmwGMi3ipjg}y?%3Z6Ea&{{qE*|r&TNi!td{We2BY?Qm7=qwRza-wtsHTD}>&Y zeSCk)PVTgNonP%FtCNVEm2mJYNXEVXZu4*Z9c4}9oS#9qY!!)6Q{+v~RS0M~0LeK= zi&OKr+ZbJXZiOXtE0geL^1}0()_PL9qK$i5sFZg*<|9ym)Y#;%ap<->T>#g5K^tm z1CXlht<^aYXRa~@Qnst=uq#|b#^N;I85u;N0r}I!d}svIVED{phFsom!p!-ly|=dy z|GcgXDP--!T4+ikq$*4CG*9%-19CPxGZBO&nB?|rDgDH!c=<{0$&{{4uICd-Xc{oc zRBzNpXL6M-aGR?jziUvpZeA8-f zq3Z&uY+VqhhkNMkzgnqOI6#jP0rN2l^?V>c6-(y;i;}{68sJN&6F*7td<3-WZVy`M zSfvTDf=F?6K(%qXA)|@Ede3KmjQfB7XSiNSpWDxWR|@{WPgcL~|G&iNLD})|0W2>b zJ1h8u%7e;-k3Rc-@bO2#5AJ;OiwKjQ@9yWmSmjfk`abvshrSO!-I?!$Pjck@;MNmg zkg76v&hBM%I(6k0Dl(YkV88Z5PO1ofE{r}mlmOz}OJv1O<$tACV@f?lnO~=X77cME zAA_Js1F1KeHpB`=<$TE~45NV*$jz)>_9mWqA@np6z0Cu|U5I1Q@pY1f3`W3*NQanj zxg_j!IRKJ^B9BggGKBC;b;d`iK80ydSlA9tVuh2XPm*}D+{c0yIu(`PrALh<8ocw< zx@hp7oQ44aexZoMXr^9zEGr61>W_o)9bQaJ#M`$N&QKmvNcXzG`aS?{d6!@SDzS}qXx!i!azmd08hIcd`g7Z zfq?70;V^CxKF7w{%~d0MhZ7d*eJvI@jKgm}5_pt#&U6Ih8V&i^^wwhEJHp6FwcOPn zU0I%I*?5wBo! z7bW?n+rYL5(w9UT6E!~o4frCpW`}{*uB$Zzr6i4}x*g-pZXQ4I6zePG9f`Z_mmVW~ zz`2Y=MO-ya)NCq@D^Jq0PE#y=r@H4~7(*i%x<9n~g@C5P@EVw*=OyoWLO3Ieb)@=0 z?Ku3WhCR*zXw1YGXzJ54W0PvKN6Wn!Pa<*;{v+vg)!5qqWv?#!u|Gnw z$ipKE+?VJUII}QJ>Oj48q!{J$cw17npsHu(N^vdn6>cek}}r(gw7PlnrKzp z3q^+Nn_ll=X~kUu6?a+e|NU+JHr(WU1&cd=c#2mf9$cMV;>J4om(P{2u@wII;EKps zt?2XfzgltsL?7J${T%ogn^IQ6C`>+e8@HYRK6&;mxBq*#vikM??=SMP?tkhHAcn5@ z+4uL}cQmOAZad`uiZ^-k6js_hMI1$ZpJFr{kJUv?fsz?@d*goS+_P!MfHrTH2XePi z&2jQsRcF08Z2vt-m*`*jY$Hpjd*9ULQuY-~R9*%cJ9&h!PT6qriao&V;l2=a@PgDTsRn1^tFyReC9$Fv4h}lVE_0OB{u#HM5R*oT{&Dx76C}?#`%iGmgE> z7kk|;^&rTlc)TGV4>eY|K{{-A{1J%rGucJ)IJSPTK?g|AzSJ))vaQ-COQo_Xobw3Q zy>sk+Fr-^mtV18stGU9+I9|*AhteP=U*MP3Hpcj`8d+H{Fvf-5XW`+$$MW*j5xKJ_ zema|Ka>_=GEW4*xK3*`zp@hWitS>e|*&W7OCjw?i<{*0GaWpul{m(D3Ip4-RtqrP! z1aMD~&$353?*2*A%dy`uTIyoL0ltc8kyB;c)Wf{;Qj$?`D&t`QC?8fvP-b^QDTcC# zZOmyJ8QWw3eE>Cn35MyJ`16iEoDb)Xk7CIfBZKC#e~P=7coUC@8z%WTDasNiADb9} zXu+HgZVe4V^0JDlRg)8Flmu)ayR#Gg+OV50;g8(>F_^@jNs?1?#SM0`(8$#VgnH;h zlM+5R64xV1(0Iua;v)=_L7=uQz0Rf$x6(RCBM0!9`I{ao*aY?N)E|~JTA`aI9k9rz zCYz)c=z3E+jq#GpaNy{l_}0MbI6CphebDJ(&$!j@9?-Mt|2S+Nw)zL1_I{_``?W6* z8d34Bi~r6x?t5Jp6_4rl5R@@TR{vjKy{wY>sWw3TYzJ|Y3KT+B*ko3d*-4hS2C`uQ4g1OgigN6Vbmzu`E;aH1#mLV@Na+dK85bXS z-tkU+{w5#wIa1ToEIIU3FBtdcl-IM>tbzuTB#Gk79^Yxo(X*+tw_xuJ+vHj7LpL#+ zMig5ysSwz0%qmy)jQ#JxrNdy@r8VPzY+((5QU)Na;e`R7D-9dJwz?lXV0+!2=+r@h z%P*lc#G{whdTzv62E6>AvuFR5ACCVNgJc>1{<+$?E&l83v(^0h@6+Y4`~NTUSzF8P z|2x6pOuPvO@BB+wzt?#pvk z4BiT0T7z*g1uzHXA+66SSMUV+mIzEfS6zg%?(}S0LvR;oh&zW2;#*$I?7-*D)A8c@ z%CdWwPR1%+71<%m<5K~^SNLcM1!J;0gJ;hnhSmqQ0}ar4v`wAFvtb##2{52-9`qI= zQD`ipjcRPH4o@MM4z}^`CLWHnrp1NW+nVAx@*yq zb{guW^$3=d-(e|nN3ocWZT(6W`rSMC+#sS-K{Th3KMbh={GC%GjiPbl$_NWo z7QW*fXccGZk3!P97InCd>byudFqo|^eyf+ltz_}SC*G9q6!jbfF4`#_3dM0*M^`=* z3*=!k$m~Tq;tqyVIl|jM{51Fyy=}S32}ap$n|Q<_96228IUD+pjk>}a(YA>|)Wn18 z+vI!0UVlb(g2om%Ht-D-U9w#Nb5ie}zX($qUv4Ic5HdJn@0L!+Uij{(e1LRa6FK66 zf!8CzrSUd-TotQx>&OT59yse_7288k&_}ho#fi9A*W4B!2_7B26X(ar;)vSCH&)_B zI+l4?gLD8SS?a1GsVI!M5CsKc!!P((CP@k_O3PG5MfvwS;|>a$X>vyg6MzF0U&tFE zAR6Lm8sjAz6+2Th2*;R}#WNmJp*bDcEiMMKp#*xWx*}`TK-5q_)bC^2XD3i+Zn2sS z3Z^Qm4~OJyfUxc>$~R7;A-?uh)zLYt=A@uQ*~LV8m9RHKJ^-C^EzsUX?8Rrv3?>36 zWu0dyTTRR&T5!Dviw*>A^?=E49|B)|?w@v}w%y3W6)C^d&x<<$PIye9wWd{F;?1F9CE zxw`Mt&(Owg`v3B3-v9IIlOMkNe}0iq{{4qlApRnq0shTTU&SCsKv#5QHzHksU_1ul zio?V=Zfla+553;e$&sB2wX(0y3(+@JE5022|6@H%0mMhUmI92s`j#@%B>eC0UCNa{ zg@38_#>9s@n97j;pTxryV0{V~Qvh>|k10JO0CawbfIuC82R!JqtBJnbXD7RU2zw9T zeLN?7I-Ws(GB;o9D0B14mj`vzVhOyYkfpwhQqXwcIII zw{uk9&M{I6b**8PX;DhOP?FbAv!wK~t%(NAd)aIk*xjdysEJSPc*TbZD%Pb0B}rW_0mD8whcyWVtv^rD$cR{pzx`Qq_LT{OeX4fS|) zGE4BT2);<56iVl07T{Gu?yn3}|BBOUDp9&UWhWlgIgbq45@G+nKY&&hxJ=~6uT0EESYn|xeEaMMn#`KDH!@Oj zUnjV`^uvAar&bw4&+;1-;orZ12U=miO9!!IFM2RFkkKSpI`kmD#frTRK)z=n z_Od&hl5}*chkKiETAP1wZS^;s7^5}cE6M5{CC5vC5d@$*EP{QCKK{hHp7G3-=8CG)G3B)PXG>!(yzwm*)*Mu5)a>E&c zs?9gNv|eht9HKU&A6LL|F#JLSbO7~K`IG`Ikdf4A-yM;;-L{65|1jD@tLV)c0w;Lu_%U8&O_4p>f#aGTGqwO{+h~sD8n?K z(C#4<-$S1?W_&VzeM3CNnpZmQTHvEn3-yJ}FXWF|gpnK0yf`bLrCn3`s!`rg2Huq4 zY}e&E?V1v?{M)xi1!bZ%D=2fNtZ;N}goz|_+weJ!;V&(jQJyJ0RXCe)X%!2yE|_xG z+4w_tV!LKSlW?P3!6v;q0^UPR#sM@*PQxdv+cHb<*Q&szGz3F`L@$);+S1{M4i<30 zMrU=g@S@@49dL~8eAwGw{IRxRw5PiTEGgUD^g>+93CMyjxX=&}zpcs!0QSq+#|J`^ zj60aZcwGzq30^?s8?ql~u;sP#Ii32xxRDLbx=QMk*_?_xX1$tSQ4IywT#=|rdp#m# zVq&Tr5}yo_QkdU=pJj&VPZ5k1$p0sn{`+n6_~FuRB_1ppE>JWefsap)ly}tVA%5_U zCxw+_$M3$w&>~})IWHBgZ`XCwRi2{Pg&6cS#0gywA324i1>iM+DfsXKw#5{e08c2X zSC57bp&zM#G_1oeI)%U0VtDZaF^fg!<2=8t?6Q?_5bWBb^&@6bn=^0ZOZ!O#;+WZnd@Tw(3-e&Vm ziV=flO5_OfkS(@fGLG3qrQf&?EM#fqffw-$f6HrGAWF<5N$v1S*3y!0iy52&C~y%K3f6G zF*VLv{ULw<=jqB<`~MgDl-U2Diw>+bZ@jclH-J>kLX3+FBnZZMf}GIv%@_=sEW_rx zfIzGK`xsm6MrxR0$;l9boO`u(#|K-J^8CAB^hm4qSzU?khK$p@BAUg8Z6B$ zO_uIa?hd{$b5lOukF^TCmr)Ddu0McS} zgB8wKOM^7tmsj!?%*3VjWG_ma6qMnX#`5)atMbjRTdMyK>YMxfMdCKcrxA?n*#teS zZz=n?U%GEdrM$gdJDkqu^te#$uBb}-zS(VV9>mdka62NhMyK1nxkBvYTvhk;0J9_t zfALRijQ zjaQH?USaZ!C2xZOC_ko$9sfMQZw;{`9@31SvcD%Eb6MKQiyp|E)DK~YW#Yb}&ZE?k z5uUvP1lWjsV|jaA?Cie=txsMhKlM}o=GfW|wTZcV9)|E&FR!U5Dzht}GL$i{W`EoI zyZ8YUJSE1-BTaj`%1f=Tg0zoo4OaS5TtqX_Fi&7JF~KK6(RdbNypn8+t7z%&0h-Bh z(bSc2NgI-JADGpZh~n_z=dsw9HGu*d+7xNoAH0+AXP#rfFJUE_;0u8<`MP@IZ@=_h z1?43@xhY=sHt6MVweY1#HR-)2S8?j8#8FjWQhk1zVbfN~Nd1|{S`Ju-sP66!L$363 z-DyVo06;`&CG-S(=fW~dT^2G_Gy8aP2hP>FoAqUvQ)WHbP|rMyR?RCg&^?U8#c(zr z+qGETx02|fCK08w)hM{6zn?_W*!R#+I!T?LNNeII5Y|rBCM0uyMns!*-NRRZYi;)W zt$((=7yu8?qvRvKp%XALWe*HyFW8r2;8Z1edK`cqWtmB;x%Lc$27ZipDGEEpwZf#NV4N%(h%P7O z5FTc_6e)or^r zzY{;;5Pu6ykoE885uT%AIIv}@a{KaRl;c8u{0Q3==7p={d*jJrJmw+SSL2hpq`%6V zEnUZKfP05KJN@=vx7Xa;RD)%eaA+p16@fA|ImcpI#Vp62v4Q1^=jl%`+^bA_(@TLZ zA?g?5mmEePAsC1d;;^$rECXAHTlhtCtJ%KmV}#(uAvxAgk{Ku-wOg2Y4mMKnmGUDX zSgvR(3-Ce}UtF^K{=KL!d<2Wn=WrSt)CxWsi{~IJ_U`LA(ffaI?e%-V9w?5tD>%F{ zbG-yNRF2kL-K<{!Rjb))b!zh`23CE+?7qQNM)*l_g5=tsQ?GQUWZCo_UDig)_#u=ojKJ? zIWAv_UysT)_g8Qq@=+1Fj}X_pQh&<>y+s5vF^`Ugmr;oLL)9pMnv+_(>ZydK<{`|N56*N?V>KFOcolYDSrK zK^fJ?-4Z-(C0u1%dWsqat;+!C-dTyNi_*-kgZ+b=-KOGZGQ^l|WusPFw3F>9iY1l0 zrtIo3GU@P>;DBPijSXe*Vtvslz2PVwPytkiTJifuVmH;}G0l~qX93q{CBbvc-X!J> znj5;btnJ(i)|>?-e~k9j*Nj!w{D6YC!6=SW=64Gtr^v*0$+(JL3+)?al|E~-zb;3$ zp@hNow;&@4U*4i>%YtMYTj(JSJWFC1=#UMdS1;)rlGV_)x;3xtgYvdC9+jazRM|b{ z1yQmRt7dxI9;(LyR%KWX73)clFjwqBC)x0duw*WV2EfqeBpI#-Q{k9eURx@iI}wLQ%><`yOeKVouE zW>nAPs*RxrxZzk3xZ58ZVYkHV8eYm_N+pIbe4~ zc|O3IiIPibR+r{`fJtaedzv~!z+Olc&2hdK(Mfg`@>w|M7k-~n%g{0javPh><&5TV zn5g&3^*Ksp9EhA7nngnseKyVpU>s~@_9O=-3!88RISDlsmqFrnjktx69F$Wj`c$z0 zA1&@^jS}a9r};rG`5cx4&)8jg5k_&yizPt8L`T`Z2^nQbx3PiLGHeQMG14<)S7mQUs!AUb z!{#pS?E!{G%@0(fI|MMU34r_^o{X+*HabfN14xeNVFd%GodSvVv2cg;Y*o1XC!yI4C2&LCOLb zV#IoHaL)m(Yvo?e#sk-Nvr}VRMrQ;u=x#e=Gnj%qHB-hL4x8g~&Lo_*RoKjNtL3`# zm!swU!T%`g{Z)iB7SCVM2PCD4GnpdIYiJPOpwNUEEHvQoVUh607+C3&)&+@gUQ4Ca zn>37B>%tMWe8z(>eIwgl+fuR2@c- zs@r<3Eh-<c=3?D zU4b4a0lrDf4zB>=JcwqAiYPUAuano>%XFAKkx475HykS8exR%`fg94(R&eC4Im?n% zFF7dJ?ax*G|3e9Fo73Mj9GGo`&UxuKbm2skB`Nz&4R*GMqa%IshhRx+1BKLB)ya{b4pF;<|d7pW*kdlKm@WQ8>8ybx}9m&rf1^alv$jrIp|6(IKW7fPq58f$q?tCnd6Z!Xx zWZogX_OViDYaek|&j!)Ec`#xi&|(vWC^exDN=@YQ5Cum(b_%y&M8WQGC;YR}$V6j2 zo~qV`U+l;odZJDb@$L+o6Yyj~KKAFBXG340GZaLcpM8*M(LvHcZik{;% z_V?pTNuqLwlU#n27%ZI2N__O)D#VMw-5s$)yRtPLM+010yJTf$(V3kM-H#&HN&7sb zPAR{UN3b^YOB0twx`kmt{GoBL4JQ_vo?l(2e&XcsE4qih?f#Foy6Uyjol^f8n<0r6 z=3b)XVXupvm$9~%n5v_N1w{(pX>m-W(y7+ zGSa4C3RvJiXucnY@51OJ#7nZXFquv9qMbf;)!++KnR}YR<^m~tD;gA7ywhzqN_hZ# zWEk1zFC};Gm3)^0&24kmmE!2-G5XfHv}nwXx?w528;+lzvF9Ux0#f^3u>#Q_jg0dL zz_9s!f&pMF<3|pFdC$}ufy{FGA2R~-Y`&cSpKF%pR>4~|M0LP%pQXoG!tcK#D;;yr zid9$>OZ`-7w>AH4N;FE2Kvd)3a-K`BtXWq+?D`8H#LK-9>rjXpC!j}v{;(@QK)0$* z<$gfXRNkc3$6)gL5hHl*DgA@#7^}{QRvtlcH+7QF9R7WXN<@E61&+7&DU>uz~6g0x! zZttO^Zh2?QJAqeWWItWpguI{AtDDiEHwAB2uz%++Gs_c?XG*QE9EIU11EXduteTl+{lL+T4m-6AK>Uof)a zZOlQ^{LG)__#vJ4KGWNvK{?;5&zvcXA5%ma<1tT4Om7noA3D($E+vzF4q6sEcb(=% z1+0WNlBOdMHY{^jUp4iwD}QxZtFNCselY&qm_w$-C!eW}TjD=I`QiC;KK}EwA6CA` zfBO=j2dL-%%Mbf}AEG}dN6K-7q-Ui(9CQa{N+LFmm~Rx4`9MeaQm3jXm*SKns|@QL z8=9ii;E_U|T)o0e3<;LtK`I4b0|Wq$8G+4uLlTyESg@l(K&QCFXfT`T`wSTLEuk23 z0HNBIb*eRLXy{Xn98v;Q7HJyM4BuPiFt7BKV-OC;vmxW53S=S52!MPK6Tr?eL>=N< zmyxXDuTPz$bu>Av1C%T-TOHl3$11d)jH;K=WA)KA@U_0v)sH$-jO{lq(cRzf{nG5T zM7t{vI{QDjw^~~&yjwjV)$La+iW@*^{qvyH>Qbn;_U^$>yR}sp?Y+&N!>#t-Yw-$d z?(O%)PJ6fA1F*e)YSIi5w+%pcTb<1}kkEY9-f8!Kt&8n;Zx5kv1K6fGXm)z-&BL8$ zM;sh>4)(h(Kw%32@3r@~JJ3#Rx3$-ED;4!7U$qYlmj3SEp zE%>Fzk(UC;%-CFT>>}^}i!~+XA`RW%ZT0|0^-&^@9wo;D{w^-UzYX~BZCK^zyflsb z@`1?)ceySa7zL5@hyDl}57NLJ2meyK*h?-5z%k@F$!5^!oQFv|1b(d9rCbb!1kF%+ zc4}}M(oE9VdAbrdjqi1T$G2#`Pk7Fcw>0+Iq=Wy|X6$gXE)^dQR(X;@? zO`y#@DESk6V)ZEF0N3|GEgMGS&%{S7NqYIGti<;p`jVU~w?<_rev)7`!gUb_V?~NT zb>mrf))7===~rG&hjdxq1z~jGZ4Eg=Zb&Olz|M=la z{{6q_t6%lMFY;NkL-ESXZZZz9)|Ik%KZMilZr5a}RMK$CC~biTzVAMpcbeFji4vwFZ6i zESOS=TD)oo*eYBSBRnwl6XX8gDe^Fa$_F3*kjTT_vn;r+xc{SrU!xEIwb6$q(UYkc zzjKp6#ya9I2~JRQ#$%tOdCFJpIFj(I*=Xd)cG=h;VOckxg|rOJ!T`NWeHRaZL7laN z`C9GL&0T)F_-YH6xCnaIoCbB#I*-QZ_!98^y3>Fkc-LV^`*X`e|3G^IMffto#}d7e zo^uD*8Q)SVrNB&G*g+2H>|H%*sMi^_3?D)1W4^!mGKGy|g3lAuYkUN(>n9vHwrl+2 zRqmzPaq&`hoa+8D^W)1FJ>W;R|6%>cHBs4N5uwCSk2=xCF^8BTEyu@!abPeLeBp3B zn}m;R4FmgiX|Kb^Kp~&bQZ9vh8rDKJ&FQk&(k=XO#8N%cZ9*>w8J+qgylf42I=^Su85!&m$7m-#F$iSEJHKNqFWv)CTO z5=_1<4XG28HVMT_48c1)KkusL%C-kDEn$@dBU*@Awmw(!7U%fX2OR=a1$|7GxCqdE z^=~{rpOE@1wO%)U&oSB6*2CB=xRuI-U=-p6N3i|oR{oDi7G| zb3+LLBCU!qw71ZPkvJHj>SLv_S=~%ZAj$Wvdc{KLK&2?&s8mp-fW(F^1ojt|lqIAC zz6wS0&sj*xS629dJ9`)qE#_=#!m4)B*MERf_>UK+ZZb6O8VdQmG$hm8i8ctPJKMHm{Y7)dIO4DlV|S*NBm zY}7&OTyW&+CH;XYBaoQTkO=18q0H6!Wb!&7~{jb_EH!H@YomFbSnFHj-bdvP#H5XV$9j}Cry;-7+wfxSWI zz^9z)MF8%hJszsp0IO^S;IG2mo8iExINR5`5KN?DKN$64r-Ed0saXfsc>IA@N%)G? zC+uX`2nu2}2S#iGi3?C%y!oyDe90M;P};y(y@w@E64mNjk&;Fqxqh8$Ytn+41AW1q ze1P>hy6|K44lSo(w1fG#5YbH6#&Gy;@FDGyP^dXvAS0{%EoqX^g_;qeC@HP4{$a73 zdp4U+Z|vD2=Kr-lBNT7$nT$e_aXNErlD!C}eS=29d)-y^gOMdM?9`lY;u&Beogpa- zXx@tHsKTisQ_99mv8DMZE|y6OlG4(7qQ%ZJIHWu z?4WGw)(#rA7VS#)@}Ie$claI{Fzy&OZf8F8P6`tb;{vHIek#FI)n`th6U)`brW=>5n{3> zFsYuT3IeOsk|M8PI&{o#tWen6_jqew7buam#w5MQCuD3@2b3*fOWc^1}~b_kUmFQ@*ilPNyI>)a#Gzm4eKnp1KnJ zIvDQr^3OQV-OqK{&<#2KUVQdY5^omGy&ZAlT{n00D{t=afxvim*xT=PH3aO;I7nz? zTHYyT+%{gjmnoak_2Ke!x> z$r~yLMJ9nQ0!A$IRKR{iRPi1U2C1Bac!Xu{5=ikoA3BJEH}xo9 z1$q;~o-lz85cdj_4-*=m73mR2XYxLkX^5W7p==~P7$83Z9z^Yg>DV)s zh_SrMwk}Pp4)F0H4o--IskE-kqUbp@&PEMCLw^KxyyCoB$hn61u^!DMXaq+?Vr2j& z4-xYC!bgAdGp`}uI#9JH9ckv!F5_a&=Jz3=c1QcSrDMk6IYK(MJCIUUzKNJgG(Ine zU%s7!RIf-krqEBHpF*Y$hi0Q@h4li}yyPd*E04pXiy8(U{T1IL91?UjN!Rn7U2;2G zj>iQDucQ}8nBijS9g8a&@@Z~Za+rm|xC|%l`!CsmJ69Z63`xnGFthQbbSVAQx{~ry z6JEwh#m!o+gpEtHCB;yUaH?92GG>}*|2>XqKI;ed|83YJ?nXm@?4~!Wz#sa#S^r;o zy1JbA|9`sj{OMQy|4V$NCi@uGIP|_|OYs7JUC=KECx76zfG09f{>rki=*%B=PWj5h zg6l59|LJ5n=?_oLqU{hwJjxN`xU9yKonclX2aZ>>V2t~Vk|t!qg_e(7Ya0``~l#Q_40-ypo>SyXr(3_ zwXNRa-hnPJ;`v*8x$B`-Q=dLhz4WXh7Dg_9S*S~i8jm3jt!OTOm&r0Voyw6vWo8o1 zV$^gvkAKOmVAx@gC;bH2Kl)N7eA^(mvOl~6LtrSddBH^pcuJ6TzvyUR>KR%q4{a3&C zyRBwt^Ua-!+Q7SG>$GCIpNG2t$g}7_K7_z>K08#TA;Q_@#E*}TZ8-!o6Po|+;gw0D z!-K76uhnnA-rMiAwwO@Y&ciE$WW0KLU0eI@UMv^S!W)TVkO2^zBW7Xa=Eo^6gNcK# zP7Quk40&TCcP7MF(V2BHNY{qLARM`&!$UnZVAar%>dKT-2GY3 zP6^pKek{%lq->$EdwVI#Y`APgH2Z1F44h5mO{_u+aUgLt@ zX>r~5o`6}1UvcJ%#;ujr^ehqH^K}>|)IEuDH*rT;Z5L&p)@?|$LA@dvnz{tjfcv(x zrlu0IOa@Uu1e_(2kE9cj!Bg=Z{sQgCmZC?;<=l1s#$!YcoNP?lH8?<z!@~adpK$24oxDD}p?YJmomFHn$#M4W})W^eP&0_~CMO+Bh#^uW0*INnOP zwiqH`tocZgd;5z9k|KbX6=MBJ&NvSpyGFU4hrf16~t02&b2ezTVq$?fyCKH2%N zQ|>RCas{Bvcbq-TcVhVE|7bQXSY_~X-XMyH^8P|yC^vg+|DT;MnU-w^{xwk8U;V-J zzfLrRY2{AOKK<1%`rPLK_vG1XKK|Da&%U1jeUZ2`=Nh9!P| zbuq;dp1wE_{0qKA#?F8UilaDC_)sq?1d&jY!|-@ZVEBD`mp^kcmOXmA8BL~92$Jb7 z&H{STzJl=_iurHSbeeetzU}z1^GMRQIi)4Ex7y3`?Jflq+KbTQ9$olZ;Ox7DK-U@U z5&$7g?hJyKI6r=S7?0)Gb_i2`1e91)xAR#FP?QA@*dbGN$%^f2N4c9i;(>S=V}jhG zszN0ViiZr0|)!T#i21mnY6dc`8d! z*XW|K+f+R&ef~m|h=}XRI|2{Uyn<5-)@tfOjt+8(mc4}qUJjO1Tijqk8HC$gl~clz ze}Ss}WlrN7Gz|b3B_s|Yy^W4JhMD=He6KB6lqZ$-xI;0Z>_a-V@yUU2U3Ip+d(=f0 zuXXNp6Auf|Gh}hi=hkux3SR=uAEqN`0sXa*12ac0iYxahsV>x2*Xzb;mZ+A+q=$BF z-}0jZdJa25cPxWSM+-9a*}`$*cH%>i#TQdPq4SFA7(}eI@aIBZ%z?6I!Ju=7AnX%Y zI9dN}(ENJSEm7p>N>wKZ?H)1XjXvDS^uT{PAw)W{opj@jGsVuFM!eyWAnGa2amXl5 z$}MOV57usyaf=%O$B}hqExTDMhRx1tVbCa(fMX%9 zo*JYnPfs}EyY1z?Y}3|SW&T(A@_on$_5Yy{6Zq0i&OTEcx9R^stgPhq|K}^KU-kbl z@pm^Nm42JP+b1L01{1_p~Vkz%K@!>>qEf9h5$ z7$IoUpG9ICOnnT3&asTcQE)nAzvPPcxa)3q+6TSU>9+Uxu)u0#@u71XAl=F9-4~0C z6h(A#kzXjOnH78O-B!2P+&v(yaS2$QiSHl%`e^cK_~^}}-A7%(nrJtUKz~BjgacD$ zAUdtB^@X>~Po5ktuRU3r{Qvf@wyA9-34iBT%xbBNyfF%py-gJa5(6gZDq>jUY$}yY zse~FIgzdHRtJdDUyTyEbgw4SUx< zIXQi+0=IUbK6L}ThdLE~T}!i3$d=lAD2WGG8C5T8hqJhO-{~Hoepvt7J-iX?KY6%Q zibODoUzSMbee?cptJk%u#QtBlbxOtoUY3*!d-I+_PU{Kza#A?WW1h?xKwmH<(rgf= zf&u6zqdk3Jhn?GuaT5x7)*op&{5Zd$i9;%W|Fr9(=+f}|0{{haB};59h!aXiinuSR z`(8Z3s$&P-`zh5T&lcXIRyoC2+{p*h^$@`ZdZp6lDCN{-~3SlE2 z;k}lTExzC^WB9K1ubRwH)L=}?_zP*})|$$HO%vlafY)L#AdcvOX3RCo?IBFY(I~En zlfMDVcF!AMQ`fSy{#7-E8?Q?LK=mTxND7+Mf>lFAr%^S**^0j#?l(60%La89fd3j!ra)^Xf;W6kO>zLrvdDm5<4>2Rbd#- zFEfTJ$>)<0ow2LmntX3A0Y(r`^r##K5|yBE&c`V zVuq#Dmm!xzjvy7CqH%~SYRg-$X6xHEaUPj*$Q2VtxAm_?Ov_(4Y>Z4CO?WbYjCfC2 z1f^Q!FMF$1!t7;1tvH`jaL!xv<*ajBl>pIR8Vui_|D74&UMCUTd%C$C$Z>lkW(+2Xj&SFXby|zpZ zq$(SwSofVCN_Fg}foP~{6aWp$Wvtq;DvX|~qTVGrBJgTTc|K_v${hzIe`~TnNDc%3yCdo6R*Ks;$j{+@uBR&x5K9%DOT#?Hcz~QB|Z;IG) z)M2*Uaeg3ycNONaI5$;rS21I+zr@Y2#0@r)L2a)WfPZ{Rxohw_F_VgQcN> z8{Kzq@dO{^Yq{H~>$W}FfRnNyKBA~?iYt_aW*FOE%BdN-V>W>=rIL4hU)=_?g@bvE>yYzM1-IYm|uLSJ* zh6p)Q?@m(p;Qt)>5wm1a=#B4zMRITtH@J{H!eSYiNrsfzj`}ZNys+E{_M2v_q$>_# zt;g>EySzWHLtp?GfV8`b&Wf@l|CO^gpG?k98<3CDrycv#4QGAYk^joX%_l?LX*Tky zE8fnMZ>6$0TDDVqE5(xXs@0n+(iiO9B(p=r>QX0P>{QOE92%wx>c=^18h>2{`@V zK03f!lI@nC4}{~T;KYP3q7)JP(P;1Y?Z(Ss($!yZllBW#b!sIp?E z%JiaPGN-CMaNamS^>dKqgLxX|{?;}zz*Pn?#Q*fX<>C1@592>e{1f9NeDIA?0PdLo zKihm-_5VUYAMAg=%i{ugsr~q%+wQv_UuwQyT&FIWy7!kPTJCe@y5w`^b*Hc68Fc6B z>|d`4G)7mRWfI1V87V_N&v`HMoY9qbOK*Wak1go;L_$Q)Zi)0NrkO}GsKhu*qR}KP zFgR91PLj!|%BMc#fk~4hYk2fJ8RQv!s|0O~Z0wFCI5KEn9S!ECtZlFqc&-dG_sthb zT+I(er_y|_1&@-8A@ig}N5c=$vVrQp1i58M?bN6@Mdv6frda{-EqZCecG-TZ9181G zWwWnuWq%}HE)qM>CHT?td@^`m&n~-wK=$`VXz*Tk??qn`5}`$gxrI>2AZ!@}qfjHX z)syjhARMw7;-O=Vl1$i7Z(5bCzl6RfZ#U`7=jJAVzzyl2G*Pt15UqT~E-}0FG_{5h z&^?;Q81fZM2rE~06o3CZ@~fn+sPCq&i1^FT_Ur9He|;6t7w!ogznA2 zO08%SPODHfh_))M4<>aM{St&!xt3rDVKkwPvV_N0)+uLkq+-^pt0LrvYvQEyao7PO zE(%B~tM${UbD>egsc&9F)~U9jpI?&@ev8t1>}J=*Az&E~Z*?{%%CfvHiqoB0DFCX~ z8EJ$nl&dPQ1p~#)6KUQ#FwH4APV!=QnyaORKg9HT7@U`d>sEc;n#rZS$2Nc~ndGCo zGH)v6kii%`3KPwtxx7i}&PVBLCADJz$TH^8yG3_xte{gGNHk@D1>ai0J1HztgOzEH z6$;wDGd;dTC(5IxtnJm*5WZWZ;lSWf;yOi|O~7_Z$0__qaEjH|!8NK!UdZqVylO+# zGh=>65lQuZ^(d#LfFYQyEC zGg)LGiKyXZg*MBM7lv#?b&CEy`&h|Dg){#U7l{QI!Q^Z)Q*|0kNj$750i`M)QRyUu@q z*{+`dK7IDf!};%bd6wFIVtb+I?&e^_1qX!*wm$eQZ9L_hMb_4{dTp%NgzpLJz!8?m zvGFGxu-jW4{N}N}w~I9TG&cvt(GU1)#5DhncFd;tR~Dw<*QDi`t&P6JRj0w=d7Wx%57 zRh{9g?W?P7s!qJAP@j`D6&Ep%gfHi5Q&3Ff9JkedM`V2+>jXUPz+8cP#aM>JHg{rE_n z#W^hZ1s3rehvU5vyw8&;>5E6SaL%5Sl+8Kv~ zEM)~}^IWA*$^@CF8VQfc19qtCq@6-nF=Qfnfunxo^UOLnGm%t!>CuF#7&mC_dBEXz z%($WYPClD8+dalLmdPJ+h%-IusZ%`PFn2TqT4B;Fa;WZ)TGkR+1fJfdo#gyzL(S`9w;n^DUP7b=E))xkKd zvjXOKn2N|(eU^1me>>= zR8m}qrd`zIUz#wS#3^}TAGlN$DcC*vUqQbb+=ntn8R`mUucU(}7ih3@&(*+bEgFAV z5vEa1nXxhz$K^=WJ5m&$qm}eMh`%JTBegM&Vj@&JsBl677pUAi5XpY;P1~rauA`7O zhPZJwUJ8~~0f_3ZPN)hsTD4bGDkZ=fHBY#l|AT4d`rT?C3u?NBtYzQmmso0wC@%aj zCjX|mb~B@LKjw@^aW)R8Su&aVQ98YX@^1!zErnjO!@+CbUS@am6s0NfQGp10!wBP_ z*VG5v!Xy3iG?6sAD5!s)F*d8$Gi6@@+Zeo2LNMydH@O}xJjt<*Z|Kc%SRRh4Pa8+6 zG;SY_ifAEWlNjZzREM4^b+{-(DUM(|8F}z-wk74beY+eBX&bdysuv?ew36`CC%^I@ zvfTa&k#upcl!{+BeidiL!2JtKd7=cSaN7+9tDF=G4K+Tu+36OCG#(aR56{E%@O;1L MPtH>%9RSh+09sxgZ~y=R literal 0 HcmV?d00001 From b9b728940ec774f87e9196ecb96dc4106163a1e5 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Wed, 11 Mar 2026 12:26:53 -0300 Subject: [PATCH 469/489] fix(winnowing):SP-4147 handle binaryornot errors gracefully in is_binary check --- CHANGELOG.md | 4 +++- src/scanoss/winnowing.py | 6 +++++- tests/data/src.tar.gz | Bin 29169 -> 0 bytes 3 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 tests/data/src.tar.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index a5babe9c..24b14e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [1.49.0] - 2026-03-11 +## [1.49.0] - 2026-03-12 ### Fixed - Fixed `--skip-headers` incorrectly identifying continuation lines inside multi-line import blocks +### Changed +- Added error handling for `is_binary` check to catch `RuntimeError` and default to treating the file as binary ## [1.48.0] - 2026-03-06 ### Added diff --git a/src/scanoss/winnowing.py b/src/scanoss/winnowing.py index 08588cb5..6af7f06f 100644 --- a/src/scanoss/winnowing.py +++ b/src/scanoss/winnowing.py @@ -294,7 +294,11 @@ def is_binary(self, path: str): :return: True if binary, False otherwise """ if path: - binary_path = is_binary(path) + try: + binary_path = is_binary(path) + except RuntimeError as e: + self.print_stderr(f'Warning: Failed to detect binary status for {path}. Assuming binary. Details:{e}') + return True if binary_path: self.print_trace(f'Detected binary file: {path}') return binary_path diff --git a/tests/data/src.tar.gz b/tests/data/src.tar.gz deleted file mode 100644 index 682c9262d953ce5551cbe481257d0fc89c12b268..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29169 zcmV(>K-j+@iwFR=hOub?1MGckW7{^e=>1y#3RFpIDwXL+9_^81H;OGM+VyKCx!qpJ zN2MjoW+Ra*N!jtbzQ6s>0{{V%@*}VAo)hghmhc!127|$1Fc>89VCnyS=Fjr-@(<6R z3Htk-|1PgS<-hVHR-Uaqd-`Pg$;y)zvAnYK!_z1KM?Cx7h~a0Jq+SfDc&D@ErgBhh zG`b;Q_Cx>u5BVf8{%2k|9Q(0*8hxHNfF94EKAk)MPnVxO;qhO6zWl?JXE6RxeprR^ zU;aGk*q{I9KjG|dM6hR1yzSwDRw)VO$F$%`MTd7RF!8`BN7iVcYtyF@^G>TKN} zr&ajshtO~ko-X~KL?I@OCX^mT_;2W^OIRDfCut1j68d@>4)AXpO#Eu4Rzc+6@I=6V zPNQ&`h%^#_;+tOY0Mh=L`AOLM18WjevCRj#SoPBcpqR+4W_OlIRWn2!DTkWCbuc8 zH!+q=AbH|VkC>Xrk`|S3!YJLQp;Rf{y*>Xzf*-`bht)k1`WL#}fTI@jg`iY^iv zCssJrBoIdy5Zqc_gHg>AvhHN&?nHxkPOb9I&>ta7VbWnZmWi>;=pRXN$6{SUvT)u; zEau=1S!oc3_v`>doZAVkU0XljV(4 zst^!cVOs)?hvIMD{XInmae-vOd5Nh~{9$sCo*AqK^E90$B;_TIO(63~%e6LXX$aH0 z7<#Ff$<3?ln(*T|qDg+?zVW>wPJKW-b*h^v7K%T_;vxwD z$rM&SvDw*Ne6qSJW(f)y6pC#SZ(}Y?Ivr|~AQ~&;v9ZH&HfylUSa~9Lz4)DIlD-24 zeg@?RRhbocMTiyHdz^A9yVw#4>c_9>F^;g05#qp)CqY8p#4QrG4j9u8<1z|; z{5GSFk=Z%b;B*#GVQHuX9MJQ}GZO1C785@n1m0K}D`QfJ8Vq<$`gKabLFo@r-8;e2 z3sL|yihaSbu^5cKV8Xi^%)E%B)F%WOJakHvPqar=ra&Kt1h{y}14H0A z?7V;0OA1I1L1C+v%!VaMt5X89g z6=D^)#Jq`;+un2L-h?{{ou5M2LQ+uUcTmg&glB^@9v6d1+@wt@EPUdtIaMSD)Oq6a z3;-Z7X`zuzMfBL8z|MP!obE{!i7%0ZB(9nXNNcIsGYnwimzUmnHVFfFfItRku0Ncq z*@x5bC2q~NP{0JxqkOUQRBZVJv>_~G+DRPvBSBVzGcO(zgMiR5MT3ClF>L#Z-v0`wb#Y4n`so0` zHZAUvV`oVb57obsHy$sZ04LS)_k|B~0si8AOr|~*LZoo=N4STE{w6*fDU!z0(kV!v z*@+89mXd)NMoF^dhcJ--(#beFS(=FnE^B;%N9Fhbl3-k**?~fUm z2P|-W@ghlwpo*St*h#QR6w*M|wbN!HbYp0zO%Vfcn6HdGQcOc4s8z-Zx|`j0O-!Tl zWf)CR;sKtqj~&M`2N@T|pxe8M@dHq)f!5++f9F@R{Qk$)(eo!KKm14ldzly)f6Tn$ z1yoz~(nW6(?Le6*1F{e-+)o4pAyMr|;lS^wlK1&nasgZSgWj7?tGU(R-`i~U+k5Ta znyNViQaoAB>ZFLk(T~SRtIwVvugNL(SWOq$@JJI(su9&VDPq919L&aY-(etm9tA_y zOb`Yspyps&t#U<@Mxz2dbtqOB@75}mfvq?pbYpo19tmKIQVsAL#))qe>3SDLM45z^>0xtp>s`JtS%296XU#AY0E@$z4H{}{sa3$1 z{H|Po&@g*~+T_YJfnoxdbI_1z!9@KfG?ZYy)R{6>!SRx0hPB5*ngSAjI1HdNngvdw zebns`fq)HK0}mBRwLoSIBWp~gq9%zyIYBcS+1+8yBV`!K?^1_Rj8Bj(foOupGeMaF z!txwecT|COiIdq#Y@tyGzKmEOmpNmgk&%PHL5ZgV(tH3i2^L;~i*+W5@-h$PkeL*K z_(3_x0_!or1|2l`7YbHrKuIeuG7FR76i*TcKv`JvVa`%fs?cJu$`Ipx4`XPJwpi3D z`2qjzyd`I|o~AVhXU>I!B;Dn+DLmnuc%Zetm$K0VX*vLvRp22Y398rLI6}b`;7%HM zjTf-p9{Z$?lGJ;>?X1>7O1AJEOXiT&CCv`EcVQ+8q`Y)-x03R9>dZR_aY*~hvG3uL z7wpAo>mLGQh!e}yDN1dH0WMQ?oPcnH5$v9!QQU}go4h#~n4%QhYlpBEksUU&prp*= z&YLn4wst@&Q$WJz5EWD{6i-L1eFoVl8fN11F(lWsf&jdr&B2-?MIU^lwcF$#tYL6Z zFrp;X9BLNlVqFv=&|bzTSaM@y-BcqW|NHMtSckB<1CPM=EJzc7JYx1xBmXK&&xpRb zWlKOaSLOs7+9#{nR2*F-=5WYj`?D13K8?t}e)DkUM}Sr!gZd28AU`Jg+bfmAuL6-h^#y+WIHrJ5;g6CS$Z&<&*BM!ip;OmvCAmmu(%Se01L}w zacpQAxf8b#s=2IHwfi)-;Ibm%EVPW!^I*lE2Zjv7hG-h-&MGJ`UT8Si^ZTH>jC~+` z1&x(NrZ}0I{LpA^Z9dFVG>dJZ24#vcA+t`+8Zbv>ijhKz?8ws@E9!Gb;4%gI?2}ns zw_K|$f~bBiF0#HL0Rzlr9yu=2M#iYKtucudvD zgW$+s4Oer8Zm>P59J+7j_|b_V-@XwCujw@AODBLTpY@ z?>vz^gG8z?Qep9uB$7u#Y?Wr4goieCrCB9Df$;`|D8@56JWoTM@T3Za&Oj+{YN}^8 z8{0r+n5_xZA6Ncs)k=B4BC42JwNqC`b>;b!AD?E)+&>sTI|b`QVy+z4f3^ZZwQZb9 zs?AtPND@TECahed4xb|@jn*9v=4baan2)3I6b%DxFP~>&I*Z~oE0vpR?J!6|Ujc?O zMKJYt)tOcfKgd%zHPl2sa3g zJfY8QQ{Jbus3jMn(^;FGh=^IT5j2<3*#dpVn?JE-?p=;y2{Z) z)g!4g#U3W4yNs1@7ioRi@mMhndD!XwFn`#SnY2w-B2LhE(2Fk>LLXx1?DxH7((gM} zqQJ@9v?{93qm^1+JX)^i3}ZqxM0Gu`;#|mSf-)2GuqmirU6#d+C6_H*b8J}ihz6Px zNNAGlI5i(XVRC_1`Q9JQq*TUZ&;T^WDXNPh@mQvWXF|HpZq@*jgdfu#G|E&S5kynq z9AJL6xAeuHf98$-iB{=al)gsf%iY1H-Uso~+s}ex&7^}n%toJjN_X5dVv&g@k8Gl)Gv(_JK8w3hA{9)Br zi)LzKj8%IJwy5|eKLdwRX3>F}fzXi`L)hKJ%%PPJXCbO^+KhyLD;`Ksn2jX5(0Y_{ zI{NeI*-NTlDf>#$AH}}}<8v~wR>khc!CF=OpUuKjJGwUy7weTUF|$XJizVCKVIi>@ z)mTYqAu1c0$6D%gaA1e}iG`=sjuJ5hH6*_oBE!-7r)Wd z2Gd@c&EdZNn{0SUV*k~*Gd}+ZeUJHMHgTtypQnwR@Bgf>E<>h$|L5uQlc!(r|9pv0 zwOXyry#Tb^?%|9!uI~(L6eOT7x|_|t{cg8-u?5`_PcGH{mO9-sK}7?F1{yo~!VB6e z5Sni}0L}&YF?Y-CH71Z3@?}?o@a5RAJ?NgHy zSi99laJ#L}<{S9geAV7*_kKmI-FCaThj6#|JEAELnw?&I^Khrx5eJ8zgMCojn|oUT zc(1*;-GO#myRE&R3++OhX#EU7U?#-Bw*RJMAu#WxKP#i`_)RLY;jA12y+r3PHX4TkX)F;a2|0v9l}hvHW_t%muD{*h>9sojpPM^}REt&41qxHp z$1ocU4ROSmI_PxJK;f8;#0z|XJy%X40);xA+_P<9Q?URfQ*%wl10sX4cz#lBei2xrFeO{HOKo9(EyTN;|q z;xW{7vWk>aL$^BE!sL?$ZVGRPDD?&#s7y=6(kS(iDloc^omsoC)45xit{aAF|NX$9rlLiEW=l!?_2@lcXuRe(+^ zwq|`3YN>RG*^8aIMfy&XQ;^&Xdy!gQXv4q@;6fR1Lxqwh0AVbc$@iB2XjmC7Tv72d zO7{^UNw3=V+c0ZOg{x6;7`d11cQWx|6E?hY1P!n8ps0I<7ruH0&wY+|Kvku5b--boy z&x^@h3OB7V7@#HMuls3%LOg)E_lE~d$WCYZzgF}K9?gbq%^dnCv(tt+iJ~zu(Y80{ z$Q=AZmDZn^`Nu5q(|MW5LsKt3YoOSG>^)dDvfNCKDbG(N+mz+WDKjFjy{Q1rvtp5& zB_fqSowWU1LzB&%CaA==%pkG}2RmOUcbO>k#-p49ibYmbTc*NI(HX3Slm~HDsOmk_D!&?ZI0Fwfs>7$ze8MDXLhPH zU*vXfa$Cep+BouteGn>rs1PQIgQK`1S4|Sot|=3pY^c|>uPQj+I{(SR%eK4g#l3Su zA+8sv+B$c&IE&V~^Ti3WPXE^D4fwk8y}S?&(Yp03D`@CsiZ;I7d27i;IvHAnj))7I z;pMR5Au@`CdSnVc>qq#K4naJrB}KEX1(? zf}ITRny9b2LCNj0@MG??kX(+cYj0iduwTN@UH43C84c*-Bz98mm$K!uWJ;8cB3ko~ zYziKi8#MHw(s+^CSwYSmTBQcAV%ZMabnnsqULB}XsbSJ~Eg=?$$=F|9^v&s>@uf{^ zbqGuoTO&tR9R;IyqY9g=L>e`9f@p=kvS`!$^aQwRWk&c8VZ%%geaclTogaiYRnXnT z6PkImLS6waSez8n%t3wGTY}@FI-58(yIs&Lb;%;HQ&x2HJ6bR^yHZ0pb3M0)JDXxG zD@Owdy|(CA#NAL9FIBg|Eb`uWZr-Q`foDvKeGv%OdQWyy8bcB{Y9z7dEklBXh{BG9 zl&VrT7};`lb#3ho#XJA9&S64u6d1v|<3Qy~#{i)p0jy)aBKq8~K8B+5`dS0&6O6>6 zKgGS5)$cj8>n@O4H#T}8;U7`1i~IShQzHuK&T~aKnT`Y4B-R&d$Juz&9X@M3?NQ7w zcqvlL)dv}?I1+cW#}+vr#(=4d<@z!vvPLVnK`w1laWa)HRfVkLg#p5n2Ah}S!R(&H zM$UaG&?lag-+95SB@ss#2T$$)zFBr2a%Y&w`7hF+voE_{8=IX_p|3;-~ag{pT$M-X9fP*jfOMWDAEUZbf8bW#1;d0Gn$;>GY3(O;s`U;5rCNHOxTJBYAEH}y^eP{J<>1cmBN$Ucaf2T`oE8{}$+ZtM~^nmZ$m#!TUX zPkm?vFQ^mGxx}qhCQO7my%)j}gR=GMk`H1X z&8@F4vu`W(6=Rg~x4{`kj5fX+$+7X3lYI*0R2TGQ2>SK;LkS!q_JT1(!muOFGv7nS zt0pecGgEtGnSDHngNblh3e<)3;?E6qpo}juWQ_lQN^6V(4VF6O8u5~eA*RVo!C}^P zL@IS4gR8V$tjKKf63b#R77W%~M=%_e25+$PQOyKHWe(ZcroOC8_fHOw4G5Fg^^yt$@&KnuYDP%ca4V{vMnm<8 zTFg4gZI^f6tCXP$y&GbPqP}>sNEz_`J$|36L@HR(I@f#EX`Wk9cv|2+IkORGyzYX% z#fwOZ^6afp6reDfO|+d8H>R=)lLRl{1Ld-z&>0A!Kpu$^P{*4~gA8lv`BVT4&gmFB z%--BF`BVX0hp>;JJcBsE8@Du|ZH#3ImALDHNe6F4E_5~tR0p8tY=BMdL1WHv!;2bI zyp*Jl$7?mSH;!gb5_V7?{%W42AX5~4G4@L)Q$0JHLLf+AteuI2%mgYoFNLw-R?GVV zdAUhx;7HRA#k2fr^a*CB*{79Ur$~Z*_CBrLuTv|!S2WG^)Ow%bH;RzyU_K1ww3f{> z=RK^V?-MN^EMaw}Y=99cw9g1iZ{>3oEEiG=FWRJ^}4Ut#6! z)D!6%*+U@L7`l?69xojG81_cqJ_ycf?i(Qk5jHx#tH;=K0~g^ejxx!Sl!*>`DGzTt z#4kRm7sILe! zj~5G+$q_{oAeW<;h3t$K(e+TSNH^8vs-7bOMb_EO}-R&0GZFj&B(!H*M z&ASAKfbM+{EO!fxqOUDrr~saR3u~1M-_^$llW6lx&ucjg_TOp*#xs+Z^jp^cwj^7x? z4)1>Wyj%q=!5QL`J!M^*d*shQ|Fn=;%SxYN$oZ@^{WMA=UPfv8-tsy&}0k6!ib)uloNN`7A9N%0C_Dpi(C4Q*Qu&c9in0UP4@Tm&J>n z_GW9Z+gfZMG&e!Xh++hS^0u+Gba8RvdQ{ns;?pH{^JGcY0RS692EhV7P+d(y2pTS? zV*q4^h&E|{IMap*r8eqz(%q<_jap>eaN-X}0rIv_sP;Sg(xi6EHwI0-1qETLwPxkT~CTGzY zA6SqllnnZ&}Vmda&?L9w987xdEf~@JP+b1#GNAn zIQL?D#TSVA;0Ty4IQN|`z6`KACWnk})>)(Hwc`rx@f7ylX({s<5= zLAUD$J*0-;@+HR6=_w3&L+tPul@E)?N1$n9V<3y`JfKQf7c&%aO=^hQ%JVDWJ`8QBm|V=BtQxsNctdWML$UZHyze`E{`6xzV2zD| zKZ&|x57?z=pM=5-Ikz^9Z0QW7s$J8u?AYclWLa{I&vChWXgZnt0}?09_&A;2tOYK} zuIzlL>^sDRY@8?*_(I8xE{KK=@Ajq%-P}ccQ7qTp)?!=q+@e;$I z{-4w*+O=@g`sd$+5M-TnpxdG=>AtkEst}LmbS7^#LxgDYK@V5O!O%~NDF(HQbgCZ}ff9j{D6>HT*O>Ew7Kh0+=zHyVI~Q|-1oKesxxQLGjo)nrXh zYNB$hiGPgntR*es{GZ#Ym%g5mzM0Irlq`aMk|~(MPb%}Oi5*y)TnWkAgZ6T{Ve)F?f*z+9?!hb(!%Zgzo$PG{r{e>eBJ+jiI3&dr``GX zVYLShA-xpsaNtTux_fTbI3}OWs=^WnAYi+sH%1&fR&AH${)gVo0ht@%Wr*xnn}kB1 zEx$?_e3E3HA?bqn7$_r+pSy1lWu?QAqIpZ>GM`2Bt_IitF$Nxqg$L4(&t3ove0`9D z-K#S!1Y>OR44Nqv%?&Y!REE8dJn#^(oVB}jW}Z9wXt(6)!QNII?ekqb5}qHRCjn5C z5APz1lQs^}Ljav9WfoUawslicK$=Y`UoaFBuk3@f}JYK ze?`5_NH5c2dj>l^k6*c3w|>eJ;VGV@!@u%3U3y=qKQhHf)ee&AuPPPESDM!KSD6Dc z_pCn}M^puWxb#EjMhUGKXp>J|{H8k4-{re;5(w%mR}?E^=p%lR4GKbk*X@In)Wvvt z>tX>`YIk{|Qc*^7DCMF)imM!Lq?E$YPt$Qt85OZ-x=2m@zjaZCwfe-5tK?B3GsFwA zf|L&R(m>tEI3Kcr`QB#|pu>caaJVTylwnrXNQ_51${T)?G@=2!GL|1;r69h{CF&N} z^;aVh1!ovbuhD08r7(U$_ns^pCC>udMHnf_e7=!(Z5gLc?vWQRCTXog9W}=Qjs2XR zU2d!ka}3C64O|zDZ5qplAiXZCSO|j+1>>qsQ0|%R5nl?)dZ~@rltDz6_aaN$)5VaA z8}jC)rngm;h>aVGzJDGJK%>hBEgV+N60RJR?ECj(#qL1?H|cDo^OzA-rkpjP_kJ&Q z7uAUYUjHf#!Kpa}HG>V+<+&UNifA&sgj7r##UJJ`)w{v~(p^Ec^jFI!1awgWGJ2~J z!CX8R&$_-rX$72MUkCdY-_ia2Kx7FAKaVO^mGdO~u7F4YJa&aKkcj1mp@cL4-#l_# z90En`T$@J<*#$0KEOwok$K+vk3z#k%7J4LEpRQ;bBN>fhf1dXFBue37Wl5QjdAyfX z7oTxR1GGd2^4hBGRgHbX^enaDFu5XHc|lH_d4(x!;RgSfWlHP-T47FWM#ykLP;lSMDBuYP7JtmFpVWA#}cJ$j3omLi-n#9C5whJ z&#ZcVbVIZz(2{V>*_4~0HMRIq4`naEx@Gh&N(lNf_ zyB3x9V;ctk4bd+H56I~HY+oiRFdS4uITv<+-R|`__jh-j9u!TzzdwSsX%ra zBBv@kacLWMIgIm_d@O<8X!$7;{2bGNqO?`b?8*bT@yfUhB=o=>3T zLaSA*U)q>_VgN0bQAh`$ifMF;E}~uxFCCEPLEEOf%0L0J(}FRP#>swn|FE;!>e{8= zY`^JmwGMi3ipjg}y?%3Z6Ea&{{qE*|r&TNi!td{We2BY?Qm7=qwRza-wtsHTD}>&Y zeSCk)PVTgNonP%FtCNVEm2mJYNXEVXZu4*Z9c4}9oS#9qY!!)6Q{+v~RS0M~0LeK= zi&OKr+ZbJXZiOXtE0geL^1}0()_PL9qK$i5sFZg*<|9ym)Y#;%ap<->T>#g5K^tm z1CXlht<^aYXRa~@Qnst=uq#|b#^N;I85u;N0r}I!d}svIVED{phFsom!p!-ly|=dy z|GcgXDP--!T4+ikq$*4CG*9%-19CPxGZBO&nB?|rDgDH!c=<{0$&{{4uICd-Xc{oc zRBzNpXL6M-aGR?jziUvpZeA8-f zq3Z&uY+VqhhkNMkzgnqOI6#jP0rN2l^?V>c6-(y;i;}{68sJN&6F*7td<3-WZVy`M zSfvTDf=F?6K(%qXA)|@Ede3KmjQfB7XSiNSpWDxWR|@{WPgcL~|G&iNLD})|0W2>b zJ1h8u%7e;-k3Rc-@bO2#5AJ;OiwKjQ@9yWmSmjfk`abvshrSO!-I?!$Pjck@;MNmg zkg76v&hBM%I(6k0Dl(YkV88Z5PO1ofE{r}mlmOz}OJv1O<$tACV@f?lnO~=X77cME zAA_Js1F1KeHpB`=<$TE~45NV*$jz)>_9mWqA@np6z0Cu|U5I1Q@pY1f3`W3*NQanj zxg_j!IRKJ^B9BggGKBC;b;d`iK80ydSlA9tVuh2XPm*}D+{c0yIu(`PrALh<8ocw< zx@hp7oQ44aexZoMXr^9zEGr61>W_o)9bQaJ#M`$N&QKmvNcXzG`aS?{d6!@SDzS}qXx!i!azmd08hIcd`g7Z zfq?70;V^CxKF7w{%~d0MhZ7d*eJvI@jKgm}5_pt#&U6Ih8V&i^^wwhEJHp6FwcOPn zU0I%I*?5wBo! z7bW?n+rYL5(w9UT6E!~o4frCpW`}{*uB$Zzr6i4}x*g-pZXQ4I6zePG9f`Z_mmVW~ zz`2Y=MO-ya)NCq@D^Jq0PE#y=r@H4~7(*i%x<9n~g@C5P@EVw*=OyoWLO3Ieb)@=0 z?Ku3WhCR*zXw1YGXzJ54W0PvKN6Wn!Pa<*;{v+vg)!5qqWv?#!u|Gnw z$ipKE+?VJUII}QJ>Oj48q!{J$cw17npsHu(N^vdn6>cek}}r(gw7PlnrKzp z3q^+Nn_ll=X~kUu6?a+e|NU+JHr(WU1&cd=c#2mf9$cMV;>J4om(P{2u@wII;EKps zt?2XfzgltsL?7J${T%ogn^IQ6C`>+e8@HYRK6&;mxBq*#vikM??=SMP?tkhHAcn5@ z+4uL}cQmOAZad`uiZ^-k6js_hMI1$ZpJFr{kJUv?fsz?@d*goS+_P!MfHrTH2XePi z&2jQsRcF08Z2vt-m*`*jY$Hpjd*9ULQuY-~R9*%cJ9&h!PT6qriao&V;l2=a@PgDTsRn1^tFyReC9$Fv4h}lVE_0OB{u#HM5R*oT{&Dx76C}?#`%iGmgE> z7kk|;^&rTlc)TGV4>eY|K{{-A{1J%rGucJ)IJSPTK?g|AzSJ))vaQ-COQo_Xobw3Q zy>sk+Fr-^mtV18stGU9+I9|*AhteP=U*MP3Hpcj`8d+H{Fvf-5XW`+$$MW*j5xKJ_ zema|Ka>_=GEW4*xK3*`zp@hWitS>e|*&W7OCjw?i<{*0GaWpul{m(D3Ip4-RtqrP! z1aMD~&$353?*2*A%dy`uTIyoL0ltc8kyB;c)Wf{;Qj$?`D&t`QC?8fvP-b^QDTcC# zZOmyJ8QWw3eE>Cn35MyJ`16iEoDb)Xk7CIfBZKC#e~P=7coUC@8z%WTDasNiADb9} zXu+HgZVe4V^0JDlRg)8Flmu)ayR#Gg+OV50;g8(>F_^@jNs?1?#SM0`(8$#VgnH;h zlM+5R64xV1(0Iua;v)=_L7=uQz0Rf$x6(RCBM0!9`I{ao*aY?N)E|~JTA`aI9k9rz zCYz)c=z3E+jq#GpaNy{l_}0MbI6CphebDJ(&$!j@9?-Mt|2S+Nw)zL1_I{_``?W6* z8d34Bi~r6x?t5Jp6_4rl5R@@TR{vjKy{wY>sWw3TYzJ|Y3KT+B*ko3d*-4hS2C`uQ4g1OgigN6Vbmzu`E;aH1#mLV@Na+dK85bXS z-tkU+{w5#wIa1ToEIIU3FBtdcl-IM>tbzuTB#Gk79^Yxo(X*+tw_xuJ+vHj7LpL#+ zMig5ysSwz0%qmy)jQ#JxrNdy@r8VPzY+((5QU)Na;e`R7D-9dJwz?lXV0+!2=+r@h z%P*lc#G{whdTzv62E6>AvuFR5ACCVNgJc>1{<+$?E&l83v(^0h@6+Y4`~NTUSzF8P z|2x6pOuPvO@BB+wzt?#pvk z4BiT0T7z*g1uzHXA+66SSMUV+mIzEfS6zg%?(}S0LvR;oh&zW2;#*$I?7-*D)A8c@ z%CdWwPR1%+71<%m<5K~^SNLcM1!J;0gJ;hnhSmqQ0}ar4v`wAFvtb##2{52-9`qI= zQD`ipjcRPH4o@MM4z}^`CLWHnrp1NW+nVAx@*yq zb{guW^$3=d-(e|nN3ocWZT(6W`rSMC+#sS-K{Th3KMbh={GC%GjiPbl$_NWo z7QW*fXccGZk3!P97InCd>byudFqo|^eyf+ltz_}SC*G9q6!jbfF4`#_3dM0*M^`=* z3*=!k$m~Tq;tqyVIl|jM{51Fyy=}S32}ap$n|Q<_96228IUD+pjk>}a(YA>|)Wn18 z+vI!0UVlb(g2om%Ht-D-U9w#Nb5ie}zX($qUv4Ic5HdJn@0L!+Uij{(e1LRa6FK66 zf!8CzrSUd-TotQx>&OT59yse_7288k&_}ho#fi9A*W4B!2_7B26X(ar;)vSCH&)_B zI+l4?gLD8SS?a1GsVI!M5CsKc!!P((CP@k_O3PG5MfvwS;|>a$X>vyg6MzF0U&tFE zAR6Lm8sjAz6+2Th2*;R}#WNmJp*bDcEiMMKp#*xWx*}`TK-5q_)bC^2XD3i+Zn2sS z3Z^Qm4~OJyfUxc>$~R7;A-?uh)zLYt=A@uQ*~LV8m9RHKJ^-C^EzsUX?8Rrv3?>36 zWu0dyTTRR&T5!Dviw*>A^?=E49|B)|?w@v}w%y3W6)C^d&x<<$PIye9wWd{F;?1F9CE zxw`Mt&(Owg`v3B3-v9IIlOMkNe}0iq{{4qlApRnq0shTTU&SCsKv#5QHzHksU_1ul zio?V=Zfla+553;e$&sB2wX(0y3(+@JE5022|6@H%0mMhUmI92s`j#@%B>eC0UCNa{ zg@38_#>9s@n97j;pTxryV0{V~Qvh>|k10JO0CawbfIuC82R!JqtBJnbXD7RU2zw9T zeLN?7I-Ws(GB;o9D0B14mj`vzVhOyYkfpwhQqXwcIII zw{uk9&M{I6b**8PX;DhOP?FbAv!wK~t%(NAd)aIk*xjdysEJSPc*TbZD%Pb0B}rW_0mD8whcyWVtv^rD$cR{pzx`Qq_LT{OeX4fS|) zGE4BT2);<56iVl07T{Gu?yn3}|BBOUDp9&UWhWlgIgbq45@G+nKY&&hxJ=~6uT0EESYn|xeEaMMn#`KDH!@Oj zUnjV`^uvAar&bw4&+;1-;orZ12U=miO9!!IFM2RFkkKSpI`kmD#frTRK)z=n z_Od&hl5}*chkKiETAP1wZS^;s7^5}cE6M5{CC5vC5d@$*EP{QCKK{hHp7G3-=8CG)G3B)PXG>!(yzwm*)*Mu5)a>E&c zs?9gNv|eht9HKU&A6LL|F#JLSbO7~K`IG`Ikdf4A-yM;;-L{65|1jD@tLV)c0w;Lu_%U8&O_4p>f#aGTGqwO{+h~sD8n?K z(C#4<-$S1?W_&VzeM3CNnpZmQTHvEn3-yJ}FXWF|gpnK0yf`bLrCn3`s!`rg2Huq4 zY}e&E?V1v?{M)xi1!bZ%D=2fNtZ;N}goz|_+weJ!;V&(jQJyJ0RXCe)X%!2yE|_xG z+4w_tV!LKSlW?P3!6v;q0^UPR#sM@*PQxdv+cHb<*Q&szGz3F`L@$);+S1{M4i<30 zMrU=g@S@@49dL~8eAwGw{IRxRw5PiTEGgUD^g>+93CMyjxX=&}zpcs!0QSq+#|J`^ zj60aZcwGzq30^?s8?ql~u;sP#Ii32xxRDLbx=QMk*_?_xX1$tSQ4IywT#=|rdp#m# zVq&Tr5}yo_QkdU=pJj&VPZ5k1$p0sn{`+n6_~FuRB_1ppE>JWefsap)ly}tVA%5_U zCxw+_$M3$w&>~})IWHBgZ`XCwRi2{Pg&6cS#0gywA324i1>iM+DfsXKw#5{e08c2X zSC57bp&zM#G_1oeI)%U0VtDZaF^fg!<2=8t?6Q?_5bWBb^&@6bn=^0ZOZ!O#;+WZnd@Tw(3-e&Vm ziV=flO5_OfkS(@fGLG3qrQf&?EM#fqffw-$f6HrGAWF<5N$v1S*3y!0iy52&C~y%K3f6G zF*VLv{ULw<=jqB<`~MgDl-U2Diw>+bZ@jclH-J>kLX3+FBnZZMf}GIv%@_=sEW_rx zfIzGK`xsm6MrxR0$;l9boO`u(#|K-J^8CAB^hm4qSzU?khK$p@BAUg8Z6B$ zO_uIa?hd{$b5lOukF^TCmr)Ddu0McS} zgB8wKOM^7tmsj!?%*3VjWG_ma6qMnX#`5)atMbjRTdMyK>YMxfMdCKcrxA?n*#teS zZz=n?U%GEdrM$gdJDkqu^te#$uBb}-zS(VV9>mdka62NhMyK1nxkBvYTvhk;0J9_t zfALRijQ zjaQH?USaZ!C2xZOC_ko$9sfMQZw;{`9@31SvcD%Eb6MKQiyp|E)DK~YW#Yb}&ZE?k z5uUvP1lWjsV|jaA?Cie=txsMhKlM}o=GfW|wTZcV9)|E&FR!U5Dzht}GL$i{W`EoI zyZ8YUJSE1-BTaj`%1f=Tg0zoo4OaS5TtqX_Fi&7JF~KK6(RdbNypn8+t7z%&0h-Bh z(bSc2NgI-JADGpZh~n_z=dsw9HGu*d+7xNoAH0+AXP#rfFJUE_;0u8<`MP@IZ@=_h z1?43@xhY=sHt6MVweY1#HR-)2S8?j8#8FjWQhk1zVbfN~Nd1|{S`Ju-sP66!L$363 z-DyVo06;`&CG-S(=fW~dT^2G_Gy8aP2hP>FoAqUvQ)WHbP|rMyR?RCg&^?U8#c(zr z+qGETx02|fCK08w)hM{6zn?_W*!R#+I!T?LNNeII5Y|rBCM0uyMns!*-NRRZYi;)W zt$((=7yu8?qvRvKp%XALWe*HyFW8r2;8Z1edK`cqWtmB;x%Lc$27ZipDGEEpwZf#NV4N%(h%P7O z5FTc_6e)or^r zzY{;;5Pu6ykoE885uT%AIIv}@a{KaRl;c8u{0Q3==7p={d*jJrJmw+SSL2hpq`%6V zEnUZKfP05KJN@=vx7Xa;RD)%eaA+p16@fA|ImcpI#Vp62v4Q1^=jl%`+^bA_(@TLZ zA?g?5mmEePAsC1d;;^$rECXAHTlhtCtJ%KmV}#(uAvxAgk{Ku-wOg2Y4mMKnmGUDX zSgvR(3-Ce}UtF^K{=KL!d<2Wn=WrSt)CxWsi{~IJ_U`LA(ffaI?e%-V9w?5tD>%F{ zbG-yNRF2kL-K<{!Rjb))b!zh`23CE+?7qQNM)*l_g5=tsQ?GQUWZCo_UDig)_#u=ojKJ? zIWAv_UysT)_g8Qq@=+1Fj}X_pQh&<>y+s5vF^`Ugmr;oLL)9pMnv+_(>ZydK<{`|N56*N?V>KFOcolYDSrK zK^fJ?-4Z-(C0u1%dWsqat;+!C-dTyNi_*-kgZ+b=-KOGZGQ^l|WusPFw3F>9iY1l0 zrtIo3GU@P>;DBPijSXe*Vtvslz2PVwPytkiTJifuVmH;}G0l~qX93q{CBbvc-X!J> znj5;btnJ(i)|>?-e~k9j*Nj!w{D6YC!6=SW=64Gtr^v*0$+(JL3+)?al|E~-zb;3$ zp@hNow;&@4U*4i>%YtMYTj(JSJWFC1=#UMdS1;)rlGV_)x;3xtgYvdC9+jazRM|b{ z1yQmRt7dxI9;(LyR%KWX73)clFjwqBC)x0duw*WV2EfqeBpI#-Q{k9eURx@iI}wLQ%><`yOeKVouE zW>nAPs*RxrxZzk3xZ58ZVYkHV8eYm_N+pIbe4~ zc|O3IiIPibR+r{`fJtaedzv~!z+Olc&2hdK(Mfg`@>w|M7k-~n%g{0javPh><&5TV zn5g&3^*Ksp9EhA7nngnseKyVpU>s~@_9O=-3!88RISDlsmqFrnjktx69F$Wj`c$z0 zA1&@^jS}a9r};rG`5cx4&)8jg5k_&yizPt8L`T`Z2^nQbx3PiLGHeQMG14<)S7mQUs!AUb z!{#pS?E!{G%@0(fI|MMU34r_^o{X+*HabfN14xeNVFd%GodSvVv2cg;Y*o1XC!yI4C2&LCOLb zV#IoHaL)m(Yvo?e#sk-Nvr}VRMrQ;u=x#e=Gnj%qHB-hL4x8g~&Lo_*RoKjNtL3`# zm!swU!T%`g{Z)iB7SCVM2PCD4GnpdIYiJPOpwNUEEHvQoVUh607+C3&)&+@gUQ4Ca zn>37B>%tMWe8z(>eIwgl+fuR2@c- zs@r<3Eh-<c=3?D zU4b4a0lrDf4zB>=JcwqAiYPUAuano>%XFAKkx475HykS8exR%`fg94(R&eC4Im?n% zFF7dJ?ax*G|3e9Fo73Mj9GGo`&UxuKbm2skB`Nz&4R*GMqa%IshhRx+1BKLB)ya{b4pF;<|d7pW*kdlKm@WQ8>8ybx}9m&rf1^alv$jrIp|6(IKW7fPq58f$q?tCnd6Z!Xx zWZogX_OViDYaek|&j!)Ec`#xi&|(vWC^exDN=@YQ5Cum(b_%y&M8WQGC;YR}$V6j2 zo~qV`U+l;odZJDb@$L+o6Yyj~KKAFBXG340GZaLcpM8*M(LvHcZik{;% z_V?pTNuqLwlU#n27%ZI2N__O)D#VMw-5s$)yRtPLM+010yJTf$(V3kM-H#&HN&7sb zPAR{UN3b^YOB0twx`kmt{GoBL4JQ_vo?l(2e&XcsE4qih?f#Foy6Uyjol^f8n<0r6 z=3b)XVXupvm$9~%n5v_N1w{(pX>m-W(y7+ zGSa4C3RvJiXucnY@51OJ#7nZXFquv9qMbf;)!++KnR}YR<^m~tD;gA7ywhzqN_hZ# zWEk1zFC};Gm3)^0&24kmmE!2-G5XfHv}nwXx?w528;+lzvF9Ux0#f^3u>#Q_jg0dL zz_9s!f&pMF<3|pFdC$}ufy{FGA2R~-Y`&cSpKF%pR>4~|M0LP%pQXoG!tcK#D;;yr zid9$>OZ`-7w>AH4N;FE2Kvd)3a-K`BtXWq+?D`8H#LK-9>rjXpC!j}v{;(@QK)0$* z<$gfXRNkc3$6)gL5hHl*DgA@#7^}{QRvtlcH+7QF9R7WXN<@E61&+7&DU>uz~6g0x! zZttO^Zh2?QJAqeWWItWpguI{AtDDiEHwAB2uz%++Gs_c?XG*QE9EIU11EXduteTl+{lL+T4m-6AK>Uof)a zZOlQ^{LG)__#vJ4KGWNvK{?;5&zvcXA5%ma<1tT4Om7noA3D($E+vzF4q6sEcb(=% z1+0WNlBOdMHY{^jUp4iwD}QxZtFNCselY&qm_w$-C!eW}TjD=I`QiC;KK}EwA6CA` zfBO=j2dL-%%Mbf}AEG}dN6K-7q-Ui(9CQa{N+LFmm~Rx4`9MeaQm3jXm*SKns|@QL z8=9ii;E_U|T)o0e3<;LtK`I4b0|Wq$8G+4uLlTyESg@l(K&QCFXfT`T`wSTLEuk23 z0HNBIb*eRLXy{Xn98v;Q7HJyM4BuPiFt7BKV-OC;vmxW53S=S52!MPK6Tr?eL>=N< zmyxXDuTPz$bu>Av1C%T-TOHl3$11d)jH;K=WA)KA@U_0v)sH$-jO{lq(cRzf{nG5T zM7t{vI{QDjw^~~&yjwjV)$La+iW@*^{qvyH>Qbn;_U^$>yR}sp?Y+&N!>#t-Yw-$d z?(O%)PJ6fA1F*e)YSIi5w+%pcTb<1}kkEY9-f8!Kt&8n;Zx5kv1K6fGXm)z-&BL8$ zM;sh>4)(h(Kw%32@3r@~JJ3#Rx3$-ED;4!7U$qYlmj3SEp zE%>Fzk(UC;%-CFT>>}^}i!~+XA`RW%ZT0|0^-&^@9wo;D{w^-UzYX~BZCK^zyflsb z@`1?)ceySa7zL5@hyDl}57NLJ2meyK*h?-5z%k@F$!5^!oQFv|1b(d9rCbb!1kF%+ zc4}}M(oE9VdAbrdjqi1T$G2#`Pk7Fcw>0+Iq=Wy|X6$gXE)^dQR(X;@? zO`y#@DESk6V)ZEF0N3|GEgMGS&%{S7NqYIGti<;p`jVU~w?<_rev)7`!gUb_V?~NT zb>mrf))7===~rG&hjdxq1z~jGZ4Eg=Zb&Olz|M=la z{{6q_t6%lMFY;NkL-ESXZZZz9)|Ik%KZMilZr5a}RMK$CC~biTzVAMpcbeFji4vwFZ6i zESOS=TD)oo*eYBSBRnwl6XX8gDe^Fa$_F3*kjTT_vn;r+xc{SrU!xEIwb6$q(UYkc zzjKp6#ya9I2~JRQ#$%tOdCFJpIFj(I*=Xd)cG=h;VOckxg|rOJ!T`NWeHRaZL7laN z`C9GL&0T)F_-YH6xCnaIoCbB#I*-QZ_!98^y3>Fkc-LV^`*X`e|3G^IMffto#}d7e zo^uD*8Q)SVrNB&G*g+2H>|H%*sMi^_3?D)1W4^!mGKGy|g3lAuYkUN(>n9vHwrl+2 zRqmzPaq&`hoa+8D^W)1FJ>W;R|6%>cHBs4N5uwCSk2=xCF^8BTEyu@!abPeLeBp3B zn}m;R4FmgiX|Kb^Kp~&bQZ9vh8rDKJ&FQk&(k=XO#8N%cZ9*>w8J+qgylf42I=^Su85!&m$7m-#F$iSEJHKNqFWv)CTO z5=_1<4XG28HVMT_48c1)KkusL%C-kDEn$@dBU*@Awmw(!7U%fX2OR=a1$|7GxCqdE z^=~{rpOE@1wO%)U&oSB6*2CB=xRuI-U=-p6N3i|oR{oDi7G| zb3+LLBCU!qw71ZPkvJHj>SLv_S=~%ZAj$Wvdc{KLK&2?&s8mp-fW(F^1ojt|lqIAC zz6wS0&sj*xS629dJ9`)qE#_=#!m4)B*MERf_>UK+ZZb6O8VdQmG$hm8i8ctPJKMHm{Y7)dIO4DlV|S*NBm zY}7&OTyW&+CH;XYBaoQTkO=18q0H6!Wb!&7~{jb_EH!H@YomFbSnFHj-bdvP#H5XV$9j}Cry;-7+wfxSWI zz^9z)MF8%hJszsp0IO^S;IG2mo8iExINR5`5KN?DKN$64r-Ed0saXfsc>IA@N%)G? zC+uX`2nu2}2S#iGi3?C%y!oyDe90M;P};y(y@w@E64mNjk&;Fqxqh8$Ytn+41AW1q ze1P>hy6|K44lSo(w1fG#5YbH6#&Gy;@FDGyP^dXvAS0{%EoqX^g_;qeC@HP4{$a73 zdp4U+Z|vD2=Kr-lBNT7$nT$e_aXNErlD!C}eS=29d)-y^gOMdM?9`lY;u&Beogpa- zXx@tHsKTisQ_99mv8DMZE|y6OlG4(7qQ%ZJIHWu z?4WGw)(#rA7VS#)@}Ie$claI{Fzy&OZf8F8P6`tb;{vHIek#FI)n`th6U)`brW=>5n{3> zFsYuT3IeOsk|M8PI&{o#tWen6_jqew7buam#w5MQCuD3@2b3*fOWc^1}~b_kUmFQ@*ilPNyI>)a#Gzm4eKnp1KnJ zIvDQr^3OQV-OqK{&<#2KUVQdY5^omGy&ZAlT{n00D{t=afxvim*xT=PH3aO;I7nz? zTHYyT+%{gjmnoak_2Ke!x> z$r~yLMJ9nQ0!A$IRKR{iRPi1U2C1Bac!Xu{5=ikoA3BJEH}xo9 z1$q;~o-lz85cdj_4-*=m73mR2XYxLkX^5W7p==~P7$83Z9z^Yg>DV)s zh_SrMwk}Pp4)F0H4o--IskE-kqUbp@&PEMCLw^KxyyCoB$hn61u^!DMXaq+?Vr2j& z4-xYC!bgAdGp`}uI#9JH9ckv!F5_a&=Jz3=c1QcSrDMk6IYK(MJCIUUzKNJgG(Ine zU%s7!RIf-krqEBHpF*Y$hi0Q@h4li}yyPd*E04pXiy8(U{T1IL91?UjN!Rn7U2;2G zj>iQDucQ}8nBijS9g8a&@@Z~Za+rm|xC|%l`!CsmJ69Z63`xnGFthQbbSVAQx{~ry z6JEwh#m!o+gpEtHCB;yUaH?92GG>}*|2>XqKI;ed|83YJ?nXm@?4~!Wz#sa#S^r;o zy1JbA|9`sj{OMQy|4V$NCi@uGIP|_|OYs7JUC=KECx76zfG09f{>rki=*%B=PWj5h zg6l59|LJ5n=?_oLqU{hwJjxN`xU9yKonclX2aZ>>V2t~Vk|t!qg_e(7Ya0``~l#Q_40-ypo>SyXr(3_ zwXNRa-hnPJ;`v*8x$B`-Q=dLhz4WXh7Dg_9S*S~i8jm3jt!OTOm&r0Voyw6vWo8o1 zV$^gvkAKOmVAx@gC;bH2Kl)N7eA^(mvOl~6LtrSddBH^pcuJ6TzvyUR>KR%q4{a3&C zyRBwt^Ua-!+Q7SG>$GCIpNG2t$g}7_K7_z>K08#TA;Q_@#E*}TZ8-!o6Po|+;gw0D z!-K76uhnnA-rMiAwwO@Y&ciE$WW0KLU0eI@UMv^S!W)TVkO2^zBW7Xa=Eo^6gNcK# zP7Quk40&TCcP7MF(V2BHNY{qLARM`&!$UnZVAar%>dKT-2GY3 zP6^pKek{%lq->$EdwVI#Y`APgH2Z1F44h5mO{_u+aUgLt@ zX>r~5o`6}1UvcJ%#;ujr^ehqH^K}>|)IEuDH*rT;Z5L&p)@?|$LA@dvnz{tjfcv(x zrlu0IOa@Uu1e_(2kE9cj!Bg=Z{sQgCmZC?;<=l1s#$!YcoNP?lH8?<z!@~adpK$24oxDD}p?YJmomFHn$#M4W})W^eP&0_~CMO+Bh#^uW0*INnOP zwiqH`tocZgd;5z9k|KbX6=MBJ&NvSpyGFU4hrf16~t02&b2ezTVq$?fyCKH2%N zQ|>RCas{Bvcbq-TcVhVE|7bQXSY_~X-XMyH^8P|yC^vg+|DT;MnU-w^{xwk8U;V-J zzfLrRY2{AOKK<1%`rPLK_vG1XKK|Da&%U1jeUZ2`=Nh9!P| zbuq;dp1wE_{0qKA#?F8UilaDC_)sq?1d&jY!|-@ZVEBD`mp^kcmOXmA8BL~92$Jb7 z&H{STzJl=_iurHSbeeetzU}z1^GMRQIi)4Ex7y3`?Jflq+KbTQ9$olZ;Ox7DK-U@U z5&$7g?hJyKI6r=S7?0)Gb_i2`1e91)xAR#FP?QA@*dbGN$%^f2N4c9i;(>S=V}jhG zszN0ViiZr0|)!T#i21mnY6dc`8d! z*XW|K+f+R&ef~m|h=}XRI|2{Uyn<5-)@tfOjt+8(mc4}qUJjO1Tijqk8HC$gl~clz ze}Ss}WlrN7Gz|b3B_s|Yy^W4JhMD=He6KB6lqZ$-xI;0Z>_a-V@yUU2U3Ip+d(=f0 zuXXNp6Auf|Gh}hi=hkux3SR=uAEqN`0sXa*12ac0iYxahsV>x2*Xzb;mZ+A+q=$BF z-}0jZdJa25cPxWSM+-9a*}`$*cH%>i#TQdPq4SFA7(}eI@aIBZ%z?6I!Ju=7AnX%Y zI9dN}(ENJSEm7p>N>wKZ?H)1XjXvDS^uT{PAw)W{opj@jGsVuFM!eyWAnGa2amXl5 z$}MOV57usyaf=%O$B}hqExTDMhRx1tVbCa(fMX%9 zo*JYnPfs}EyY1z?Y}3|SW&T(A@_on$_5Yy{6Zq0i&OTEcx9R^stgPhq|K}^KU-kbl z@pm^Nm42JP+b1L01{1_p~Vkz%K@!>>qEf9h5$ z7$IoUpG9ICOnnT3&asTcQE)nAzvPPcxa)3q+6TSU>9+Uxu)u0#@u71XAl=F9-4~0C z6h(A#kzXjOnH78O-B!2P+&v(yaS2$QiSHl%`e^cK_~^}}-A7%(nrJtUKz~BjgacD$ zAUdtB^@X>~Po5ktuRU3r{Qvf@wyA9-34iBT%xbBNyfF%py-gJa5(6gZDq>jUY$}yY zse~FIgzdHRtJdDUyTyEbgw4SUx< zIXQi+0=IUbK6L}ThdLE~T}!i3$d=lAD2WGG8C5T8hqJhO-{~Hoepvt7J-iX?KY6%Q zibODoUzSMbee?cptJk%u#QtBlbxOtoUY3*!d-I+_PU{Kza#A?WW1h?xKwmH<(rgf= zf&u6zqdk3Jhn?GuaT5x7)*op&{5Zd$i9;%W|Fr9(=+f}|0{{haB};59h!aXiinuSR z`(8Z3s$&P-`zh5T&lcXIRyoC2+{p*h^$@`ZdZp6lDCN{-~3SlE2 z;k}lTExzC^WB9K1ubRwH)L=}?_zP*})|$$HO%vlafY)L#AdcvOX3RCo?IBFY(I~En zlfMDVcF!AMQ`fSy{#7-E8?Q?LK=mTxND7+Mf>lFAr%^S**^0j#?l(60%La89fd3j!ra)^Xf;W6kO>zLrvdDm5<4>2Rbd#- zFEfTJ$>)<0ow2LmntX3A0Y(r`^r##K5|yBE&c`V zVuq#Dmm!xzjvy7CqH%~SYRg-$X6xHEaUPj*$Q2VtxAm_?Ov_(4Y>Z4CO?WbYjCfC2 z1f^Q!FMF$1!t7;1tvH`jaL!xv<*ajBl>pIR8Vui_|D74&UMCUTd%C$C$Z>lkW(+2Xj&SFXby|zpZ zq$(SwSofVCN_Fg}foP~{6aWp$Wvtq;DvX|~qTVGrBJgTTc|K_v${hzIe`~TnNDc%3yCdo6R*Ks;$j{+@uBR&x5K9%DOT#?Hcz~QB|Z;IG) z)M2*Uaeg3ycNONaI5$;rS21I+zr@Y2#0@r)L2a)WfPZ{Rxohw_F_VgQcN> z8{Kzq@dO{^Yq{H~>$W}FfRnNyKBA~?iYt_aW*FOE%BdN-V>W>=rIL4hU)=_?g@bvE>yYzM1-IYm|uLSJ* zh6p)Q?@m(p;Qt)>5wm1a=#B4zMRITtH@J{H!eSYiNrsfzj`}ZNys+E{_M2v_q$>_# zt;g>EySzWHLtp?GfV8`b&Wf@l|CO^gpG?k98<3CDrycv#4QGAYk^joX%_l?LX*Tky zE8fnMZ>6$0TDDVqE5(xXs@0n+(iiO9B(p=r>QX0P>{QOE92%wx>c=^18h>2{`@V zK03f!lI@nC4}{~T;KYP3q7)JP(P;1Y?Z(Ss($!yZllBW#b!sIp?E z%JiaPGN-CMaNamS^>dKqgLxX|{?;}zz*Pn?#Q*fX<>C1@592>e{1f9NeDIA?0PdLo zKihm-_5VUYAMAg=%i{ugsr~q%+wQv_UuwQyT&FIWy7!kPTJCe@y5w`^b*Hc68Fc6B z>|d`4G)7mRWfI1V87V_N&v`HMoY9qbOK*Wak1go;L_$Q)Zi)0NrkO}GsKhu*qR}KP zFgR91PLj!|%BMc#fk~4hYk2fJ8RQv!s|0O~Z0wFCI5KEn9S!ECtZlFqc&-dG_sthb zT+I(er_y|_1&@-8A@ig}N5c=$vVrQp1i58M?bN6@Mdv6frda{-EqZCecG-TZ9181G zWwWnuWq%}HE)qM>CHT?td@^`m&n~-wK=$`VXz*Tk??qn`5}`$gxrI>2AZ!@}qfjHX z)syjhARMw7;-O=Vl1$i7Z(5bCzl6RfZ#U`7=jJAVzzyl2G*Pt15UqT~E-}0FG_{5h z&^?;Q81fZM2rE~06o3CZ@~fn+sPCq&i1^FT_Ur9He|;6t7w!ogznA2 zO08%SPODHfh_))M4<>aM{St&!xt3rDVKkwPvV_N0)+uLkq+-^pt0LrvYvQEyao7PO zE(%B~tM${UbD>egsc&9F)~U9jpI?&@ev8t1>}J=*Az&E~Z*?{%%CfvHiqoB0DFCX~ z8EJ$nl&dPQ1p~#)6KUQ#FwH4APV!=QnyaORKg9HT7@U`d>sEc;n#rZS$2Nc~ndGCo zGH)v6kii%`3KPwtxx7i}&PVBLCADJz$TH^8yG3_xte{gGNHk@D1>ai0J1HztgOzEH z6$;wDGd;dTC(5IxtnJm*5WZWZ;lSWf;yOi|O~7_Z$0__qaEjH|!8NK!UdZqVylO+# zGh=>65lQuZ^(d#LfFYQyEC zGg)LGiKyXZg*MBM7lv#?b&CEy`&h|Dg){#U7l{QI!Q^Z)Q*|0kNj$750i`M)QRyUu@q z*{+`dK7IDf!};%bd6wFIVtb+I?&e^_1qX!*wm$eQZ9L_hMb_4{dTp%NgzpLJz!8?m zvGFGxu-jW4{N}N}w~I9TG&cvt(GU1)#5DhncFd;tR~Dw<*QDi`t&P6JRj0w=d7Wx%57 zRh{9g?W?P7s!qJAP@j`D6&Ep%gfHi5Q&3Ff9JkedM`V2+>jXUPz+8cP#aM>JHg{rE_n z#W^hZ1s3rehvU5vyw8&;>5E6SaL%5Sl+8Kv~ zEM)~}^IWA*$^@CF8VQfc19qtCq@6-nF=Qfnfunxo^UOLnGm%t!>CuF#7&mC_dBEXz z%($WYPClD8+dalLmdPJ+h%-IusZ%`PFn2TqT4B;Fa;WZ)TGkR+1fJfdo#gyzL(S`9w;n^DUP7b=E))xkKd zvjXOKn2N|(eU^1me>>= zR8m}qrd`zIUz#wS#3^}TAGlN$DcC*vUqQbb+=ntn8R`mUucU(}7ih3@&(*+bEgFAV z5vEa1nXxhz$K^=WJ5m&$qm}eMh`%JTBegM&Vj@&JsBl677pUAi5XpY;P1~rauA`7O zhPZJwUJ8~~0f_3ZPN)hsTD4bGDkZ=fHBY#l|AT4d`rT?C3u?NBtYzQmmso0wC@%aj zCjX|mb~B@LKjw@^aW)R8Su&aVQ98YX@^1!zErnjO!@+CbUS@am6s0NfQGp10!wBP_ z*VG5v!Xy3iG?6sAD5!s)F*d8$Gi6@@+Zeo2LNMydH@O}xJjt<*Z|Kc%SRRh4Pa8+6 zG;SY_ifAEWlNjZzREM4^b+{-(DUM(|8F}z-wk74beY+eBX&bdysuv?ew36`CC%^I@ zvfTa&k#upcl!{+BeidiL!2JtKd7=cSaN7+9tDF=G4K+Tu+36OCG#(aR56{E%@O;1L MPtH>%9RSh+09sxgZ~y=R From 74a7aba13ce01742a10deeb72785f6e3e33b7725 Mon Sep 17 00:00:00 2001 From: eeisegn <44410969+eeisegn@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:07:36 +0100 Subject: [PATCH 470/489] skip results file is errors encountered --- CHANGELOG.md | 5 +++++ src/scanoss/__init__.py | 2 +- src/scanoss/scanner.py | 5 ++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24b14e25..8070d5e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.49.1] - 2026-03-17 +### Fixed +- When an error occurs during the scan, do not write a partial scan result file. Leave it empty. + ## [1.49.0] - 2026-03-12 ### Fixed - Fixed `--skip-headers` incorrectly identifying continuation lines inside multi-line import blocks @@ -837,3 +841,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.47.0]: https://github.com/scanoss/scanoss.py/compare/v1.46.0...v1.47.0 [1.48.0]: https://github.com/scanoss/scanoss.py/compare/v1.47.0...v1.48.0 [1.49.0]: https://github.com/scanoss/scanoss.py/compare/v1.48.0...v1.49.0 +[1.49.1]: https://github.com/scanoss/scanoss.py/compare/v1.49.0...v1.49.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 352a43ce..53b6e6c7 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.49.0' +__version__ = '1.49.1' diff --git a/src/scanoss/scanner.py b/src/scanoss/scanner.py index 4140dbff..1467b328 100644 --- a/src/scanoss/scanner.py +++ b/src/scanoss/scanner.py @@ -598,7 +598,10 @@ def __finish_scan_threaded(self, file_map: Optional[Dict[Any, Any]] = None) -> b self.print_stderr('Warning: Dependency analysis ran into some trouble.') success = False dep_responses = self.threaded_deps.responses - + # If anything fails during scanning or decoration, then do not produce a results file + if not success: + self.print_stderr('Error: Scanning analysis ran into some trouble. Not producing results file.') + return success raw_scan_results = self._merge_scan_results(scan_responses, dep_responses, file_map) if self.post_processor: From 7974777f193229247695475a6f5f4bd1e41b7046 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 17 Mar 2026 12:49:34 -0300 Subject: [PATCH 471/489] fix(deps):SP-4165 avoid requirement field lost during dependency decoration --- CHANGELOG.md | 6 +++ src/scanoss/__init__.py | 2 +- src/scanoss/scancodedeps.py | 13 ++++- tests/test_dependency_requirement.py | 76 ++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 tests/test_dependency_requirement.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8070d5e0..1739c118 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.50.0] - 2026-03-17 +### Fixed +- Fixed `requirement` field being lost during dependency decoration in scan command +- Sanitized scancode `extracted_requirement` to strip redundant package name prefix (e.g., `gtest==1.17.0` → `1.17.0`) + ## [1.49.1] - 2026-03-17 ### Fixed - When an error occurs during the scan, do not write a partial scan result file. Leave it empty. @@ -842,3 +847,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.48.0]: https://github.com/scanoss/scanoss.py/compare/v1.47.0...v1.48.0 [1.49.0]: https://github.com/scanoss/scanoss.py/compare/v1.48.0...v1.49.0 [1.49.1]: https://github.com/scanoss/scanoss.py/compare/v1.49.0...v1.49.1 +[1.50.0]: https://github.com/scanoss/scanoss.py/compare/v1.49.1...v1.50.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 53b6e6c7..e8468dcf 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.49.1' +__version__ = '1.50.0' diff --git a/src/scanoss/scancodedeps.py b/src/scanoss/scancodedeps.py index 267b1a5e..492ff2b3 100644 --- a/src/scanoss/scancodedeps.py +++ b/src/scanoss/scancodedeps.py @@ -24,12 +24,17 @@ import json import os.path +import re import subprocess from pathspec import GitIgnoreSpec from .scanossbase import ScanossBase +# Regex to strip package name prefix from a requirement, keeping only the version specifier. +# e.g. 'gtest==1.17.0' -> '==1.17.0', 'boost>=1.83.0' -> '>=1.83.0', '^4.18.0' -> '^4.18.0' +REQUIREMENT_NAME_PREFIX_RE = re.compile(r'^\s*[^<>=!^]+\s*(?===|!=|>=|<=|=|>|<|\^)') + class ScancodeDeps(ScanossBase): """ @@ -134,8 +139,12 @@ def produce_from_json(self, data: json) -> dict: # noqa: PLR0912 if not rq or rq == '': rq = d.get('requirement') # scancode format 1.0 # skip requirement if it ends with the purl (i.e. exact version) or if it's local (file) - if rq and rq != '' and not dp.endswith(rq) and not rq.startswith('file:'): - dp_data['requirement'] = rq + if rq and rq != '': + # Strip any prefix data before a version comparator + rq = REQUIREMENT_NAME_PREFIX_RE.sub('', rq) + # skip if it ends with the purl (exact version) or is local (file) + if not dp.endswith(rq) and not rq.startswith('file:'): + dp_data['requirement'] = rq # Gets dependency scope scope = d.get('scope') diff --git a/tests/test_dependency_requirement.py b/tests/test_dependency_requirement.py new file mode 100644 index 00000000..571dd62d --- /dev/null +++ b/tests/test_dependency_requirement.py @@ -0,0 +1,76 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2026, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import os +import unittest + +from scanoss.scancodedeps import REQUIREMENT_NAME_PREFIX_RE, ScancodeDeps + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _sanitize(rq): + return REQUIREMENT_NAME_PREFIX_RE.sub('', rq) + + +class TestSanitizeRequirement(unittest.TestCase): + """Test the regex that strips package names from requirements""" + + def test_pip_exact_version(self): + """pip exact match: strip name, keep '=='""" + self.assertEqual(_sanitize('gtest==1.17.0'), '==1.17.0') + + def test_pip_less_equal(self): + self.assertEqual(_sanitize('boost<=1.83.0'), '<=1.83.0') + + def test_pip_greater_equal(self): + self.assertEqual(_sanitize('requests>=2.25.1'), '>=2.25.1') + + def test_pip_range(self): + self.assertEqual(_sanitize('requests>=2.25.1,<3'), '>=2.25.1,<3') + + def test_pip_not_equal(self): + self.assertEqual(_sanitize('foo!=1.0'), '!=1.0') + + def test_npm_caret_unchanged(self): + """npm ^: no operator after name, unchanged""" + self.assertEqual(_sanitize('^4.18.0'), '^4.18.0') + + def test_npm_tilde_unchanged(self): + self.assertEqual(_sanitize('~4.18.0'), '~4.18.0') + + def test_npm_greater_equal(self): + self.assertEqual(_sanitize('>=1.0.0'), '>=1.0.0') + + def test_bare_version_unchanged(self): + """Plain version number with no operator: unchanged""" + self.assertEqual(_sanitize('1.17.0'), '1.17.0') + + def test_name_prefix_of_another_package(self): + """'requests-toolbelt>=1.0' strips full name, not partial""" + self.assertEqual(_sanitize('requests-toolbelt>=1.0'), '>=1.0') + + +if __name__ == '__main__': + unittest.main() From c82c74fb0aa8a1bf52066520663e55d6e79e08a4 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Wed, 4 Mar 2026 15:42:36 +0100 Subject: [PATCH 472/489] fix(postprocessor): ensure `license` field in BOM replace rules is applied correctly --- CHANGELOG.md | 2 + src/scanoss/scanpostprocessor.py | 44 +++++++------ tests/test_scan_post_processor.py | 104 ++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1739c118..34606c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed +- Fixed `bom.replace` rules with a `license` field: the license is now applied to the replaced result instead of being silently dropped ## [1.50.0] - 2026-03-17 ### Fixed diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index f06def20..058aef81 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -22,7 +22,7 @@ THE SOFTWARE. """ -from typing import List, Tuple +from typing import List, Optional from packageurl import PackageURL from packageurl.contrib import purl2url @@ -117,7 +117,7 @@ def post_process(self): ) return self.results self._remove_dismissed_files() - self._replace_purls() + self._apply_replace_rules() return self.results def _remove_dismissed_files(self): @@ -133,9 +133,9 @@ def _remove_dismissed_files(self): if not self._should_remove_result(result_path, result, to_remove_entries) } - def _replace_purls(self): + def _apply_replace_rules(self): """ - Replace purls in the results based on the SCANOSS settings file + Apply BOM replace rules from the SCANOSS settings file to the scan results """ to_replace_entries = self.scanoss_settings.get_bom_replace() if not to_replace_entries: @@ -143,31 +143,32 @@ def _replace_purls(self): for result_path, result in self.results.items(): entry = result[0] if isinstance(result, list) else result - should_replace, to_replace_with_purl = self._should_replace_result(result_path, entry, to_replace_entries) - if should_replace: - self.results[result_path] = [self._update_replaced_result(entry, to_replace_with_purl)] + replace_rule = self._find_replace_rule(result_path, entry, to_replace_entries) + if replace_rule: + self.results[result_path] = [self._apply_replace_rule(entry, replace_rule)] - def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> dict: + def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: """ Update the result with the new purl and component information if available, otherwise removes the old component information Args: result (dict): The result to update - to_replace_with_purl (str): The purl to replace with + replace_rule (ReplaceRule): The replace rule to apply Returns: dict: Updated result """ - if self.component_info_map.get(to_replace_with_purl): - result.update(self.component_info_map[to_replace_with_purl]) + if self.component_info_map.get(replace_rule.replace_with): + result.update(self.component_info_map[replace_rule.replace_with]) else: try: - new_component = PackageURL.from_string(to_replace_with_purl).to_dict() - new_component_url = purl2url.get_repo_url(to_replace_with_purl) + new_component = PackageURL.from_string(replace_rule.replace_with).to_dict() + new_component_url = purl2url.get_repo_url(replace_rule.replace_with) except RuntimeError: self.print_stderr( - f"ERROR: Issue while replacing: Invalid PURL '{to_replace_with_purl}' in settings file. Skipping." + f"ERROR: Issue while replacing: Invalid PURL '{replace_rule.replace_with}'" + ' in settings file. Skipping.' ) return result @@ -176,6 +177,8 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di result['vendor'] = new_component.get('namespace') result.pop('licenses', None) + if replace_rule.license: + result['licenses'] = [{'name': replace_rule.license}] result.pop('file', None) result.pop('file_hash', None) result.pop('file_url', None) @@ -187,14 +190,14 @@ def _update_replaced_result(self, result: dict, to_replace_with_purl: str) -> di result.pop('url_stats', None) result.pop('version', None) - result['purl'] = [to_replace_with_purl] + result['purl'] = [replace_rule.replace_with] result['status'] = 'identified' return result - def _should_replace_result( + def _find_replace_rule( self, result_path: str, result: dict, to_replace_entries: List[ReplaceRule] - ) -> Tuple[bool, str]: + ) -> Optional[ReplaceRule]: """ Check if a result should be replaced based on the SCANOSS settings. Uses priority-based matching: most specific rule wins. @@ -205,16 +208,15 @@ def _should_replace_result( to_replace_entries (List[ReplaceRule]): Replace rules from the settings file Returns: - bool: True if the result should be replaced, False otherwise - str: The purl to replace with + Optional[ReplaceRule]: The matching replace rule, or None if no match """ result_purls = result.get('purl', []) match = find_best_match(result_path, result_purls, to_replace_entries) if match and isinstance(match, ReplaceRule) and match.replace_with: if self.debug: self._print_message(result_path, result_purls, match, 'Replacing') - return True, match.replace_with - return False, None + return match + return None def _should_remove_result(self, result_path: str, result: dict, to_remove_entries: List[BomEntry]) -> bool: """ diff --git a/tests/test_scan_post_processor.py b/tests/test_scan_post_processor.py index 376be6af..74f13c73 100644 --- a/tests/test_scan_post_processor.py +++ b/tests/test_scan_post_processor.py @@ -22,7 +22,9 @@ THE SOFTWARE. """ +import json import os +import tempfile import unittest from pathlib import Path @@ -40,6 +42,21 @@ class MyTestCase(unittest.TestCase): scan_settings = ScanossSettings(filepath=scan_settings_path) post_processor = ScanPostProcessor(scan_settings) + result_json_path = Path(script_dir, 'data', 'result.json').resolve() + + def _load_result_data(self): + """Load result.json fixture, returning a fresh dict each time.""" + with open(self.result_json_path) as f: + return json.load(f) + + def _make_processor(self, settings_data): + """Create a ScanPostProcessor from inline settings data, returns (processor, path).""" + f = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + json.dump(settings_data, f) + f.close() + settings = ScanossSettings(filepath=Path(f.name)) + return ScanPostProcessor(settings), f.name + def test_remove_files(self): """ Should remove component if matches path and purl @@ -114,5 +131,92 @@ def test_replace_purls_purl_match(self): ) + def test_replace_purls_with_license(self): + """Should apply the license from the replace rule to the result""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c', + 'replace_with': 'pkg:github/scanoss/replacement', + 'license': 'Apache-2.0', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + entry = processed['inc/json.h'][0] + self.assertEqual(entry['purl'], ['pkg:github/scanoss/replacement']) + self.assertEqual(entry['licenses'], [{'name': 'Apache-2.0'}]) + finally: + os.unlink(path) + + def test_replace_purls_without_license(self): + """Should remove licenses when replace rule has no license field""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c', + 'replace_with': 'pkg:github/scanoss/replacement', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + entry = processed['inc/json.h'][0] + self.assertEqual(entry['purl'], ['pkg:github/scanoss/replacement']) + self.assertNotIn('licenses', entry) + finally: + os.unlink(path) + + def test_replace_with_realistic_result(self): + """Should replace a full realistic scan result and strip old metadata""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c', + 'replace_with': 'pkg:github/scanoss/replacement', + 'license': 'GPL-2.0-only', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + entry = processed['inc/json.h'][0] + self.assertEqual(entry['purl'], ['pkg:github/scanoss/replacement']) + self.assertEqual(entry['component'], 'replacement') + self.assertEqual(entry['vendor'], 'scanoss') + self.assertEqual(entry['status'], 'identified') + self.assertEqual(entry['licenses'], [{'name': 'GPL-2.0-only'}]) + # Old metadata should be stripped + for field in ('file', 'file_hash', 'file_url', 'latest', 'release_date', + 'source_hash', 'url_hash', 'url_stats', 'version'): + self.assertNotIn(field, entry) + finally: + os.unlink(path) + + def test_replace_purl_with_version_no_match_unversioned_result(self): + """Should NOT replace when rule has purl@version but result has no version""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c@1.3.3', + 'replace_with': 'pkg:github/scanoss/replacement@2.0.0', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + entry = processed['inc/json.h'][0] + # Should remain unchanged — result purl has no version + self.assertEqual(entry['purl'], ['pkg:github/scanoss/scanner.c']) + self.assertEqual(entry['status'], 'pending') + finally: + os.unlink(path) + + if __name__ == '__main__': unittest.main() From d04aa62142a10e0c374c356f87ca7c6076bf7243 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Thu, 12 Mar 2026 15:18:04 +0100 Subject: [PATCH 473/489] fix(postprocessor): preserve per-file fields during BOM replace, handle license overrides properly --- src/scanoss/scanpostprocessor.py | 14 ++++-- tests/test_scan_post_processor.py | 80 +++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 058aef81..a6a6b7c7 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -160,7 +160,12 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: dict: Updated result """ if self.component_info_map.get(replace_rule.replace_with): + # Preserve per-file fields that are specific to the scanned file + per_file_keys = ('file', 'file_hash', 'file_url', 'source_hash', 'url_hash', + 'lines', 'oss_lines', 'matched') + preserved = {k: result[k] for k in per_file_keys if k in result} result.update(self.component_info_map[replace_rule.replace_with]) + result.update(preserved) else: try: new_component = PackageURL.from_string(replace_rule.replace_with).to_dict() @@ -176,9 +181,6 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: result['url'] = new_component_url result['vendor'] = new_component.get('namespace') - result.pop('licenses', None) - if replace_rule.license: - result['licenses'] = [{'name': replace_rule.license}] result.pop('file', None) result.pop('file_hash', None) result.pop('file_url', None) @@ -187,9 +189,13 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: result.pop('source_hash', None) result.pop('url_hash', None) result.pop('url_stats', None) - result.pop('url_stats', None) result.pop('version', None) + if replace_rule.license: + result['licenses'] = [{'name': replace_rule.license}] + elif not self.component_info_map.get(replace_rule.replace_with): + result.pop('licenses', None) + result['purl'] = [replace_rule.replace_with] result['status'] = 'identified' diff --git a/tests/test_scan_post_processor.py b/tests/test_scan_post_processor.py index 74f13c73..4ababbc6 100644 --- a/tests/test_scan_post_processor.py +++ b/tests/test_scan_post_processor.py @@ -197,6 +197,86 @@ def test_replace_with_realistic_result(self): finally: os.unlink(path) + def test_replace_with_existing_purl_preserves_per_file_fields(self): + """When replace_with target exists in scan results (component_info_map), + per-file fields from the original result must be preserved, not clobbered + by values from the component_info_map entry.""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c', + 'replace_with': 'pkg:github/scanoss/engine', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + # inc/json.h originally matched scanner.c and should now be replaced + # with engine, but per-file fields must stay from the original result + entry = processed['inc/json.h'][0] + self.assertEqual(entry['purl'], ['pkg:github/scanoss/engine']) + self.assertEqual(entry['status'], 'identified') + # Per-file fields must be from the ORIGINAL result (inc/json.h), not + # from the component_info_map entry (which came from a different file) + self.assertEqual(entry['file'], 'scanner.c-1.3.3/external/inc/json.h') + self.assertEqual(entry['file_hash'], 'e91a03b850651dd56dd979ba92668a19') + self.assertEqual(entry['source_hash'], 'e91a03b850651dd56dd979ba92668a19') + self.assertEqual(entry['lines'], 'all') + self.assertEqual(entry['matched'], '100%') + self.assertEqual(entry['oss_lines'], 'all') + # Component-level fields should come from the engine entry + self.assertEqual(entry['component'], 'engine') + self.assertEqual(entry['vendor'], 'scanoss') + finally: + os.unlink(path) + + def test_replace_with_existing_purl_applies_license_override(self): + """When replace_with target exists in component_info_map AND the replace + rule has a license, the license override must be applied.""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c', + 'replace_with': 'pkg:github/scanoss/engine', + 'license': 'GPL-3.0-only', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + entry = processed['inc/json.h'][0] + self.assertEqual(entry['purl'], ['pkg:github/scanoss/engine']) + # License override must take effect even though component_info_map + # has its own licenses for engine + self.assertEqual(entry['licenses'], [{'name': 'GPL-3.0-only'}]) + finally: + os.unlink(path) + + def test_replace_with_existing_purl_keeps_component_licenses_when_no_override(self): + """When replace_with target exists in component_info_map and no license + override is specified, the component's licenses from the map are kept.""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c', + 'replace_with': 'pkg:github/scanoss/engine', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + entry = processed['inc/json.h'][0] + self.assertEqual(entry['purl'], ['pkg:github/scanoss/engine']) + # Without a license override, the entry should have the engine's + # original licenses from the component_info_map + self.assertIn('licenses', entry) + self.assertTrue(len(entry['licenses']) > 0) + finally: + os.unlink(path) + def test_replace_purl_with_version_no_match_unversioned_result(self): """Should NOT replace when rule has purl@version but result has no version""" processor, path = self._make_processor({ From 8e6c2c1c0631c6b01f3e646d8da64b9d92f6ac1b Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Fri, 13 Mar 2026 09:41:44 +0100 Subject: [PATCH 474/489] fix(postprocessor): handle ValueError in replace_with PURL parsing --- src/scanoss/scanpostprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index a6a6b7c7..79811820 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -170,7 +170,7 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: try: new_component = PackageURL.from_string(replace_rule.replace_with).to_dict() new_component_url = purl2url.get_repo_url(replace_rule.replace_with) - except RuntimeError: + except (ValueError, RuntimeError): self.print_stderr( f"ERROR: Issue while replacing: Invalid PURL '{replace_rule.replace_with}'" ' in settings file. Skipping.' From 83c25a827ca32a15b758f37c6f717b6346fa1823 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 17 Mar 2026 17:29:07 +0100 Subject: [PATCH 475/489] fix(postprocessor): adjust BOM replace logic to copy only component-level fields --- src/scanoss/scanpostprocessor.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 79811820..813c84bb 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -160,12 +160,13 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: dict: Updated result """ if self.component_info_map.get(replace_rule.replace_with): - # Preserve per-file fields that are specific to the scanned file - per_file_keys = ('file', 'file_hash', 'file_url', 'source_hash', 'url_hash', - 'lines', 'oss_lines', 'matched') - preserved = {k: result[k] for k in per_file_keys if k in result} - result.update(self.component_info_map[replace_rule.replace_with]) - result.update(preserved) + # Only copy component-level fields from the map entry, leaving + # per-file fields (file, file_hash, lines, matched, etc.) untouched. + source = self.component_info_map[replace_rule.replace_with] + for key in ('component', 'vendor', 'url', 'version', 'latest', + 'release_date', 'licenses', 'url_stats'): + if key in source: + result[key] = source[key] else: try: new_component = PackageURL.from_string(replace_rule.replace_with).to_dict() From ae0a6fa6fa538d162e4077c2dee2fdaee0116262 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 17 Mar 2026 17:55:54 +0100 Subject: [PATCH 476/489] test(postprocessor): improve BOM replace test coverage and clarity --- tests/test_scan_post_processor.py | 62 +++++++++++++++---------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/tests/test_scan_post_processor.py b/tests/test_scan_post_processor.py index 4ababbc6..92c604e8 100644 --- a/tests/test_scan_post_processor.py +++ b/tests/test_scan_post_processor.py @@ -152,7 +152,9 @@ def test_replace_purls_with_license(self): os.unlink(path) def test_replace_purls_without_license(self): - """Should remove licenses when replace rule has no license field""" + """When replace_with PURL is NOT in scan results and the rule has no + license override, licenses should be removed entirely — we have no + license info for the unknown replacement component.""" processor, path = self._make_processor({ 'bom': { 'replace': [{ @@ -166,6 +168,8 @@ def test_replace_purls_without_license(self): entry = processed['inc/json.h'][0] self.assertEqual(entry['purl'], ['pkg:github/scanoss/replacement']) + # 'replacement' is not in scan results, so there's no license info + # to copy — the original component's licenses must be stripped self.assertNotIn('licenses', entry) finally: os.unlink(path) @@ -199,8 +203,7 @@ def test_replace_with_realistic_result(self): def test_replace_with_existing_purl_preserves_per_file_fields(self): """When replace_with target exists in scan results (component_info_map), - per-file fields from the original result must be preserved, not clobbered - by values from the component_info_map entry.""" + per-file fields must be preserved and component-level fields copied.""" processor, path = self._make_processor({ 'bom': { 'replace': [{ @@ -212,8 +215,6 @@ def test_replace_with_existing_purl_preserves_per_file_fields(self): try: processed = processor.load_results(self._load_result_data()).post_process() - # inc/json.h originally matched scanner.c and should now be replaced - # with engine, but per-file fields must stay from the original result entry = processed['inc/json.h'][0] self.assertEqual(entry['purl'], ['pkg:github/scanoss/engine']) self.assertEqual(entry['status'], 'identified') @@ -228,15 +229,21 @@ def test_replace_with_existing_purl_preserves_per_file_fields(self): # Component-level fields should come from the engine entry self.assertEqual(entry['component'], 'engine') self.assertEqual(entry['vendor'], 'scanoss') + # Without a license override, the component's licenses are kept + self.assertIn('licenses', entry) + self.assertTrue(len(entry['licenses']) > 0) finally: os.unlink(path) - def test_replace_with_existing_purl_applies_license_override(self): - """When replace_with target exists in component_info_map AND the replace - rule has a license, the license override must be applied.""" + def test_replace_path_scoped_with_existing_purl_and_license_override(self): + """Reproduce Sean's bug: path-scoped replace rule where replace_with PURL + already exists in results from a different file. Per-file fields must be + preserved, license override must be applied, and files outside the path + must not be affected.""" processor, path = self._make_processor({ 'bom': { 'replace': [{ + 'path': 'src/', 'purl': 'pkg:github/scanoss/scanner.c', 'replace_with': 'pkg:github/scanoss/engine', 'license': 'GPL-3.0-only', @@ -246,34 +253,23 @@ def test_replace_with_existing_purl_applies_license_override(self): try: processed = processor.load_results(self._load_result_data()).post_process() - entry = processed['inc/json.h'][0] + # src/json.c is under src/ and matches scanner.c → should be replaced + entry = processed['src/json.c'][0] self.assertEqual(entry['purl'], ['pkg:github/scanoss/engine']) - # License override must take effect even though component_info_map - # has its own licenses for engine + self.assertEqual(entry['status'], 'identified') self.assertEqual(entry['licenses'], [{'name': 'GPL-3.0-only'}]) - finally: - os.unlink(path) - - def test_replace_with_existing_purl_keeps_component_licenses_when_no_override(self): - """When replace_with target exists in component_info_map and no license - override is specified, the component's licenses from the map are kept.""" - processor, path = self._make_processor({ - 'bom': { - 'replace': [{ - 'purl': 'pkg:github/scanoss/scanner.c', - 'replace_with': 'pkg:github/scanoss/engine', - }] - } - }) - try: - processed = processor.load_results(self._load_result_data()).post_process() + # Per-file fields must be from src/json.c, not from the engine entry + self.assertEqual(entry['file'], 'scanner.c-1.3.3/external/src/json.c') + self.assertEqual(entry['file_hash'], '8e4d433c1547b59681379e9fe9960546') + self.assertEqual(entry['source_hash'], '8e4d433c1547b59681379e9fe9960546') + # Component-level fields from engine + self.assertEqual(entry['component'], 'engine') + self.assertEqual(entry['vendor'], 'scanoss') - entry = processed['inc/json.h'][0] - self.assertEqual(entry['purl'], ['pkg:github/scanoss/engine']) - # Without a license override, the entry should have the engine's - # original licenses from the component_info_map - self.assertIn('licenses', entry) - self.assertTrue(len(entry['licenses']) > 0) + # inc/json.h is NOT under src/ → should remain unchanged + inc_entry = processed['inc/json.h'][0] + self.assertEqual(inc_entry['purl'], ['pkg:github/scanoss/scanner.c']) + self.assertEqual(inc_entry['status'], 'pending') finally: os.unlink(path) From 39168239f7f88b7be714859ea8cbfc06a3a321bf Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Tue, 17 Mar 2026 18:02:09 +0100 Subject: [PATCH 477/489] test(postprocessor): verify license replacement with empty list in BOM replace logic --- tests/test_scan_post_processor.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_scan_post_processor.py b/tests/test_scan_post_processor.py index 92c604e8..58a5455c 100644 --- a/tests/test_scan_post_processor.py +++ b/tests/test_scan_post_processor.py @@ -235,6 +235,30 @@ def test_replace_with_existing_purl_preserves_per_file_fields(self): finally: os.unlink(path) + def test_replace_with_existing_purl_empty_licenses_clears_original(self): + """When replace_with PURL exists in scan results but has an empty + licenses list, the original component's licenses must be replaced + with the empty list — not left stale.""" + processor, path = self._make_processor({ + 'bom': { + 'replace': [{ + 'purl': 'pkg:github/scanoss/scanner.c', + 'replace_with': 'pkg:github/scanoss/jenkins-pipeline-example', + }] + } + }) + try: + processed = processor.load_results(self._load_result_data()).post_process() + + # inc/json.h originally had BSD-2-Clause + GPL-2.0-only licenses; + # jenkins-pipeline-example (from inc/log.c) has licenses: [] + entry = processed['inc/json.h'][0] + self.assertEqual(entry['purl'], ['pkg:github/scanoss/jenkins-pipeline-example']) + # Original licenses must NOT remain — replaced with empty list + self.assertEqual(entry['licenses'], []) + finally: + os.unlink(path) + def test_replace_path_scoped_with_existing_purl_and_license_override(self): """Reproduce Sean's bug: path-scoped replace rule where replace_with PURL already exists in results from a different file. Per-file fields must be From b91eea9d6559f8e9415ee585f8e7f1f1fee48857 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Wed, 18 Mar 2026 14:17:17 +0100 Subject: [PATCH 478/489] fix(postprocessor): extend BOM replace logic to copy additional component-level fields --- src/scanoss/scanpostprocessor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 813c84bb..260fce9b 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -163,8 +163,10 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: # Only copy component-level fields from the map entry, leaving # per-file fields (file, file_hash, lines, matched, etc.) untouched. source = self.component_info_map[replace_rule.replace_with] - for key in ('component', 'vendor', 'url', 'version', 'latest', - 'release_date', 'licenses', 'url_stats'): + for key in ('component', 'vendor', 'url', 'url_hash', 'version', 'latest', + 'release_date', 'licenses', 'url_stats', 'cryptography', + 'vulnerabilities', 'provenance', 'dependencies', 'health', + 'quality'): if key in source: result[key] = source[key] else: From 761507278bb3bb1fbb280367e26bf09f17e0cd14 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Wed, 18 Mar 2026 14:24:16 +0100 Subject: [PATCH 479/489] fix(postprocessor): refactor BOM replace logic with reusable component-level fields --- src/scanoss/scanpostprocessor.py | 31 ++++++++++++++++++------------- tests/test_scan_post_processor.py | 8 ++++++-- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index 260fce9b..fa337800 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -30,6 +30,13 @@ from .scanoss_settings import BomEntry, ReplaceRule, ScanossSettings, find_best_match from .scanossbase import ScanossBase +COMPONENT_LEVEL_FIELDS = ( + 'component', 'vendor', 'url', 'url_hash', 'version', 'latest', + 'release_date', 'licenses', 'url_stats', 'cryptography', + 'vulnerabilities', 'provenance', 'dependencies', 'health', + 'quality', +) + def _get_match_type_message(result_path: str, bom_entry: BomEntry, action: str) -> str: """ @@ -163,10 +170,7 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: # Only copy component-level fields from the map entry, leaving # per-file fields (file, file_hash, lines, matched, etc.) untouched. source = self.component_info_map[replace_rule.replace_with] - for key in ('component', 'vendor', 'url', 'url_hash', 'version', 'latest', - 'release_date', 'licenses', 'url_stats', 'cryptography', - 'vulnerabilities', 'provenance', 'dependencies', 'health', - 'quality'): + for key in COMPONENT_LEVEL_FIELDS: if key in source: result[key] = source[key] else: @@ -180,19 +184,20 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: ) return result - result['component'] = new_component.get('name') - result['url'] = new_component_url - result['vendor'] = new_component.get('namespace') + # Pop all stale component-level fields first + for key in COMPONENT_LEVEL_FIELDS: + result.pop(key, None) + # Pop stale KB match fields (remote file info) result.pop('file', None) result.pop('file_hash', None) result.pop('file_url', None) - result.pop('latest', None) - result.pop('release_date', None) - result.pop('source_hash', None) - result.pop('url_hash', None) - result.pop('url_stats', None) - result.pop('version', None) + + # Set what we know from the PURL + result['component'] = new_component.get('name') + result['url'] = new_component_url + result['vendor'] = new_component.get('namespace') + if replace_rule.license: result['licenses'] = [{'name': replace_rule.license}] diff --git a/tests/test_scan_post_processor.py b/tests/test_scan_post_processor.py index 58a5455c..63698c58 100644 --- a/tests/test_scan_post_processor.py +++ b/tests/test_scan_post_processor.py @@ -194,9 +194,13 @@ def test_replace_with_realistic_result(self): self.assertEqual(entry['vendor'], 'scanoss') self.assertEqual(entry['status'], 'identified') self.assertEqual(entry['licenses'], [{'name': 'GPL-2.0-only'}]) - # Old metadata should be stripped + # source_hash belongs to the local scanned file and must be preserved + self.assertIn('source_hash', entry) + # Old component/KB metadata should be stripped for field in ('file', 'file_hash', 'file_url', 'latest', 'release_date', - 'source_hash', 'url_hash', 'url_stats', 'version'): + 'url_hash', 'url_stats', 'version', 'cryptography', + 'vulnerabilities', 'provenance', 'dependencies', 'health', + 'quality'): self.assertNotIn(field, entry) finally: os.unlink(path) From 4bfda8eb1cf9c8814beb21ad4b83a20c94aaaf00 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 23 Mar 2026 15:12:58 +0100 Subject: [PATCH 480/489] fix(postprocessor): reset component-level fields to defaults during BOM replace, update tests --- src/scanoss/scanpostprocessor.py | 23 ++++++++++++++--------- tests/test_scan_post_processor.py | 27 ++++++++++++++++++--------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/scanoss/scanpostprocessor.py b/src/scanoss/scanpostprocessor.py index fa337800..182b17eb 100644 --- a/src/scanoss/scanpostprocessor.py +++ b/src/scanoss/scanpostprocessor.py @@ -184,14 +184,19 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: ) return result - # Pop all stale component-level fields first - for key in COMPONENT_LEVEL_FIELDS: - result.pop(key, None) - - # Pop stale KB match fields (remote file info) - result.pop('file', None) - result.pop('file_hash', None) - result.pop('file_url', None) + # Reset component-level fields to defaults + result['licenses'] = [] + result['cryptography'] = [] + result['dependencies'] = [] + result['quality'] = [] + result['vulnerabilities'] = [] + result['health'] = {} + result['provenance'] = '' + result['latest'] = '' + result['release_date'] = '' + result['version'] = '' + result['url_hash'] = '' + result['url_stats'] = {} # Set what we know from the PURL result['component'] = new_component.get('name') @@ -202,7 +207,7 @@ def _apply_replace_rule(self, result: dict, replace_rule: ReplaceRule) -> dict: if replace_rule.license: result['licenses'] = [{'name': replace_rule.license}] elif not self.component_info_map.get(replace_rule.replace_with): - result.pop('licenses', None) + result['licenses'] = [] result['purl'] = [replace_rule.replace_with] result['status'] = 'identified' diff --git a/tests/test_scan_post_processor.py b/tests/test_scan_post_processor.py index 63698c58..e4598e87 100644 --- a/tests/test_scan_post_processor.py +++ b/tests/test_scan_post_processor.py @@ -169,8 +169,8 @@ def test_replace_purls_without_license(self): entry = processed['inc/json.h'][0] self.assertEqual(entry['purl'], ['pkg:github/scanoss/replacement']) # 'replacement' is not in scan results, so there's no license info - # to copy — the original component's licenses must be stripped - self.assertNotIn('licenses', entry) + # to copy — the original component's licenses must be reset to default + self.assertEqual(entry['licenses'], []) finally: os.unlink(path) @@ -194,14 +194,23 @@ def test_replace_with_realistic_result(self): self.assertEqual(entry['vendor'], 'scanoss') self.assertEqual(entry['status'], 'identified') self.assertEqual(entry['licenses'], [{'name': 'GPL-2.0-only'}]) - # source_hash belongs to the local scanned file and must be preserved + # File-related fields must be preserved self.assertIn('source_hash', entry) - # Old component/KB metadata should be stripped - for field in ('file', 'file_hash', 'file_url', 'latest', 'release_date', - 'url_hash', 'url_stats', 'version', 'cryptography', - 'vulnerabilities', 'provenance', 'dependencies', 'health', - 'quality'): - self.assertNotIn(field, entry) + self.assertIn('file', entry) + self.assertIn('file_hash', entry) + self.assertIn('file_url', entry) + # Component-level fields should be reset to defaults + self.assertEqual(entry['cryptography'], []) + self.assertEqual(entry['dependencies'], []) + self.assertEqual(entry['quality'], []) + self.assertEqual(entry['vulnerabilities'], []) + self.assertEqual(entry['health'], {}) + self.assertEqual(entry['provenance'], '') + self.assertEqual(entry['latest'], '') + self.assertEqual(entry['release_date'], '') + self.assertEqual(entry['version'], '') + self.assertEqual(entry['url_hash'], '') + self.assertEqual(entry['url_stats'], {}) finally: os.unlink(path) From 8d75044d3da0bdfdab8466b6b7464ded8db17e12 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Mon, 23 Mar 2026 15:32:54 +0100 Subject: [PATCH 481/489] chore(release): bump version to 1.50.1 --- CHANGELOG.md | 2 ++ src/scanoss/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34606c2b..3a464f0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [1.50.1] - 2026-03-23 ### Fixed - Fixed `bom.replace` rules with a `license` field: the license is now applied to the replaced result instead of being silently dropped diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index e8468dcf..d68def05 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.50.0' +__version__ = '1.50.1' From bb47ab19ca414f0f7c1c0d60d7bf788cd1f88fa1 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Tue, 24 Mar 2026 12:26:18 -0300 Subject: [PATCH 482/489] feat(hfh):SP-4181 implement raw output format for folder hashing --- CHANGELOG.md | 4 + src/scanoss/cli.py | 2 +- src/scanoss/scanners/scanner_hfh.py | 187 +++++++++++++++++- tests/test_scanner_hfh.py | 284 ++++++++++++++++++++++++++++ 4 files changed, 474 insertions(+), 3 deletions(-) create mode 100644 tests/test_scanner_hfh.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a464f0f..8704b501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Added `--format raw` option to `folder-scan` command to export HFH results in snippet-scanner JSON format + - Expands directory-level HFH results into per-file entries keyed by relative file path + - Assigns each file to the most specific matching `path_id` (deepest directory match wins) ## [1.50.1] - 2026-03-23 ### Fixed diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 03055ac2..29282f8d 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -988,7 +988,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 '--format', '-f', type=str, - choices=['json', 'cyclonedx'], + choices=['json', 'cyclonedx', 'raw'], default='json', help='Result output format (optional - default: json)', ) diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index 739a8921..b5c430e9 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -22,11 +22,15 @@ THE SOFTWARE. """ +import hashlib import json +import os import threading import time -from typing import Dict, Optional +from pathlib import Path +from typing import Dict, List, Optional, Tuple +from packageurl.contrib import purl2url from progress.spinner import Spinner from scanoss.constants import ( @@ -163,6 +167,13 @@ class ScannerHFHPresenter(AbstractPresenter): """ def __init__(self, scanner: ScannerHFH, **kwargs): + """ + Initialize the presenter. + + Args: + scanner (ScannerHFH): The HFH scanner instance containing scan results and file filters. + **kwargs: Additional arguments passed to AbstractPresenter (debug, trace, quiet, etc.). + """ super().__init__(**kwargs) self.scanner = scanner @@ -249,4 +260,176 @@ def _format_csv_output(self) -> str: raise NotImplementedError('CSV output is not implemented') def _format_raw_output(self) -> str: - raise NotImplementedError('Raw output is not implemented') + """ + Convert HFH scan results into snippet-scanner JSON format. + + Expands directory-level HFH results into per-file entries keyed by + relative file path, matching the structure returned by the snippet scanner. + For each file, computes the MD5 hash and constructs the file_url using + the API base URL from the scanner config. + + Returns: + str: A JSON string with the snippet-scanner format, or '{}' if no results. + """ + if not self.scanner.scan_results or 'results' not in self.scanner.scan_results: + return '{}' + + hfh_results = self.scanner.scan_results.get('results', []) + if not hfh_results: + return '{}' + + # Collect best-match component info per path_id + path_components = self._extract_best_components(hfh_results) + if not path_components: + return '{}' + + # Get all filtered files once (relative paths to scan_dir) + all_files = self.scanner.file_filters.get_filtered_files_from_folder(self.scanner.scan_dir) + + # Sort path_ids by depth (deepest first) so most-specific match wins. + # Root path '.' is always last (-1), others sort by separator count then path length. + # Example with path_ids: ['.', 'external', 'project-1.0', 'project-1.0/src/lib'] + # Sorted result: ['project-1.0/src/lib', 'project-1.0', 'external', '.'] + # - 'project-1.0/src/lib' (depth 2) claims its files first + # - 'project-1.0' (depth 0, len 11) claims remaining files under it + # - 'external' (depth 0, len 8) claims external/ files + # - '.' (root, always last) picks up everything else + sorted_path_ids = sorted( + path_components.keys(), + key=lambda p: (-1, 0) if p == '.' else (p.count(os.sep), len(p)), + reverse=True, + ) + + output = {} + claimed_files = set() + scan_dir = Path(self.scanner.scan_dir).resolve() + + for path_id in sorted_path_ids: + component, best_version = path_components[path_id] + for file_path in all_files: + if file_path in claimed_files: + continue + if not self._file_matches_path_id(file_path, path_id): + continue + + claimed_files.add(file_path) + # Path.__truediv__ (/) joins paths using the correct OS separator + file_hash = self._compute_file_md5(scan_dir / file_path) + api_url = self.scanner.client.orig_url or '' + entry = self._build_file_match_entry(component, best_version, file_path, file_hash, api_url) + output[file_path] = [entry] + + return json.dumps(output, indent=2) + + @staticmethod + def _extract_best_components(hfh_results: List[Dict]) -> Dict[str, Tuple[Dict, Dict]]: + """ + Extract the best-match component and version for each path_id from HFH results. + + Filters for components with order == 1 (best match) and takes their first version. + Results without a qualifying component or without versions are skipped. + + Args: + hfh_results (List[Dict]): The 'results' list from the HFH API response. + + Returns: + Dict[str, Tuple[Dict, Dict]]: A dict mapping path_id to (component, best_version). + """ + path_components = {} + for result in hfh_results: + path_id = result.get('path_id', '.') + components = result.get('components', []) + best = [c for c in components if c.get('order') == 1] + if not best: + continue + component = best[0] + versions = component.get('versions', []) + if not versions: + continue + path_components[path_id] = (component, versions[0]) + return path_components + + @staticmethod + def _file_matches_path_id(file_path: str, path_id: str) -> bool: + """ + Check if a file path belongs under a given path_id directory. + + Both file_path and path_id are relative to the scan root directory. + A path_id of '.' matches all files (root directory). + + Args: + file_path (str): Relative file path from the scan root. + path_id (str): Relative directory path from the HFH result. + + Returns: + bool: True if the file is under the given path_id directory. + """ + if path_id == '.': + return True + # file_path and path_id are both relative to scan_dir + return file_path == path_id or file_path.startswith(path_id + os.sep) + + def _compute_file_md5(self, file_path: Path) -> str: + """ + Compute the MD5 hash of a file's contents. + + Uses the same approach as the snippet scanner (winnowing.py) to ensure + consistent file_hash values across scan types. + + Args: + file_path (Path): Absolute path to the file. + + Returns: + str: The MD5 hex digest, or an empty string if the file cannot be read. + """ + try: + return hashlib.md5(file_path.read_bytes()).hexdigest() + except (OSError, IOError) as e: + self.base.print_stderr(f'Warning: Failed to compute MD5 for {file_path}: {e}') + return '' + + @staticmethod + def _build_file_match_entry( + component: Dict, best_version: Dict, file_path: str, file_hash: str, base_url: str, + ) -> Dict: + """ + Build a snippet-scanner-compatible result entry from an HFH component. + + Maps HFH component fields to the standard scan result format. Fields not + available from HFH (url_hash, release_date, licenses) are included as empty + values since downstream validators require them. + + Args: + component (Dict): The HFH component with purl, name, vendor fields. + best_version (Dict): The top version entry with version and score fields. + file_path (str): Relative file path from the scan root directory. + file_hash (str): Pre-computed MD5 hash of the local file. + base_url (str): API base URL used to construct the file_url field. + + Returns: + Dict: A result entry compatible with the snippet-scanner JSON format. + """ + purl = component.get('purl', '') + version = best_version.get('version', '') + + url = purl2url.get_repo_url(purl) if purl else '' + return { + 'id': 'file', + 'matched': '100%', + 'purl': [purl], + 'component': component.get('name', ''), + 'vendor': component.get('vendor', ''), + 'version': version, + 'latest': version, + 'url': url or '', + 'file': file_path, + 'file_hash': file_hash, + 'file_url': f'{base_url}/file_contents/{file_hash}', + 'source_hash': file_hash, + 'url_hash': '', + 'release_date': '', + 'licenses': [], + 'lines': 'all', + 'oss_lines': 'all', + 'status': 'pending', + } diff --git a/tests/test_scanner_hfh.py b/tests/test_scanner_hfh.py new file mode 100644 index 00000000..b5d2bb81 --- /dev/null +++ b/tests/test_scanner_hfh.py @@ -0,0 +1,284 @@ +""" +SPDX-License-Identifier: MIT + + Copyright (c) 2026, SCANOSS + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +""" + +import hashlib +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from scanoss.scanners.scanner_hfh import ScannerHFHPresenter + + +class TestExtractBestComponents(unittest.TestCase): + """Tests for ScannerHFHPresenter._extract_best_components""" + + def test_single_result_with_best_component(self): + hfh_results = [ + { + 'path_id': 'src/lib', + 'components': [ + { + 'order': 1, + 'name': 'best-comp', + 'versions': [{'version': '1.0.0', 'score': 95}], + }, + { + 'order': 2, + 'name': 'other-comp', + 'versions': [{'version': '2.0.0', 'score': 50}], + }, + ], + } + ] + result = ScannerHFHPresenter._extract_best_components(hfh_results) + self.assertIn('src/lib', result) + component, version = result['src/lib'] + self.assertEqual(component['name'], 'best-comp') + self.assertEqual(version['version'], '1.0.0') + + def test_no_order_1_component_skipped(self): + hfh_results = [ + { + 'path_id': 'src/lib', + 'components': [ + {'order': 2, 'name': 'comp', 'versions': [{'version': '1.0.0'}]}, + ], + } + ] + result = ScannerHFHPresenter._extract_best_components(hfh_results) + self.assertEqual(result, {}) + + def test_empty_components_skipped(self): + hfh_results = [{'path_id': 'src/lib', 'components': []}] + result = ScannerHFHPresenter._extract_best_components(hfh_results) + self.assertEqual(result, {}) + + def test_component_without_versions_skipped(self): + hfh_results = [ + { + 'path_id': 'src/lib', + 'components': [{'order': 1, 'name': 'comp', 'versions': []}], + } + ] + result = ScannerHFHPresenter._extract_best_components(hfh_results) + self.assertEqual(result, {}) + + def test_default_path_id_is_dot(self): + hfh_results = [ + { + 'components': [ + {'order': 1, 'name': 'comp', 'versions': [{'version': '1.0'}]}, + ], + } + ] + result = ScannerHFHPresenter._extract_best_components(hfh_results) + self.assertIn('.', result) + + def test_multiple_results(self): + hfh_results = [ + { + 'path_id': 'a', + 'components': [ + {'order': 1, 'name': 'comp-a', 'versions': [{'version': '1.0'}]}, + ], + }, + { + 'path_id': 'b', + 'components': [ + {'order': 1, 'name': 'comp-b', 'versions': [{'version': '2.0'}]}, + ], + }, + ] + result = ScannerHFHPresenter._extract_best_components(hfh_results) + self.assertEqual(len(result), 2) + self.assertEqual(result['a'][0]['name'], 'comp-a') + self.assertEqual(result['b'][0]['name'], 'comp-b') + + def test_empty_results_list(self): + result = ScannerHFHPresenter._extract_best_components([]) + self.assertEqual(result, {}) + + def test_first_version_is_selected(self): + hfh_results = [ + { + 'path_id': '.', + 'components': [ + { + 'order': 1, + 'name': 'comp', + 'versions': [ + {'version': '3.0', 'score': 100}, + {'version': '2.0', 'score': 80}, + ], + }, + ], + } + ] + result = ScannerHFHPresenter._extract_best_components(hfh_results) + _, version = result['.'] + self.assertEqual(version['version'], '3.0') + + +class TestFileMatchesPathId(unittest.TestCase): + """Tests for ScannerHFHPresenter._file_matches_path_id""" + + def test_root_path_matches_all(self): + self.assertTrue(ScannerHFHPresenter._file_matches_path_id('any/file.py', '.')) + + def test_exact_match(self): + self.assertTrue(ScannerHFHPresenter._file_matches_path_id('src/lib', 'src/lib')) + + def test_file_under_path_id(self): + self.assertTrue( + ScannerHFHPresenter._file_matches_path_id(f'src/lib{os.sep}file.py', 'src/lib') + ) + + def test_file_not_under_path_id(self): + self.assertFalse(ScannerHFHPresenter._file_matches_path_id('other/file.py', 'src/lib')) + + def test_partial_prefix_no_match(self): + # 'src/library' should NOT match path_id 'src/lib' + self.assertFalse(ScannerHFHPresenter._file_matches_path_id('src/library/file.py', 'src/lib')) + + def test_empty_file_path(self): + self.assertFalse(ScannerHFHPresenter._file_matches_path_id('', 'src/lib')) + + def test_root_path_matches_nested(self): + self.assertTrue(ScannerHFHPresenter._file_matches_path_id('a/b/c/d.py', '.')) + + +class TestComputeFileMd5(unittest.TestCase): + """Tests for ScannerHFHPresenter._compute_file_md5""" + + def _make_presenter(self): + mock_scanner = MagicMock() + return ScannerHFHPresenter(mock_scanner) + + def test_correct_md5(self): + presenter = self._make_presenter() + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(b'hello world') + f.flush() + path = Path(f.name) + try: + expected = hashlib.md5(b'hello world').hexdigest() + self.assertEqual(presenter._compute_file_md5(path), expected) + finally: + os.unlink(path) + + def test_empty_file(self): + presenter = self._make_presenter() + with tempfile.NamedTemporaryFile(delete=False) as f: + path = Path(f.name) + try: + expected = hashlib.md5(b'').hexdigest() + self.assertEqual(presenter._compute_file_md5(path), expected) + finally: + os.unlink(path) + + def test_nonexistent_file_returns_empty(self): + presenter = self._make_presenter() + path = Path('/nonexistent/file/that/does/not/exist.txt') + self.assertEqual(presenter._compute_file_md5(path), '') + + +class TestBuildFileMatchEntry(unittest.TestCase): + """Tests for ScannerHFHPresenter._build_file_match_entry""" + + @patch('scanoss.scanners.scanner_hfh.purl2url') + def test_basic_entry(self, mock_purl2url): + mock_purl2url.get_repo_url.return_value = 'https://github.com/vendor/comp' + component = {'purl': 'pkg:github/vendor/comp', 'name': 'comp', 'vendor': 'vendor'} + best_version = {'version': '1.0.0', 'licenses': [{'name': 'MIT'}]} + + entry = ScannerHFHPresenter._build_file_match_entry( + component, best_version, 'src/file.py', 'abc123', 'https://api.example.com' + ) + + self.assertEqual(entry['id'], 'file') + self.assertEqual(entry['matched'], '100%') + self.assertEqual(entry['purl'], ['pkg:github/vendor/comp']) + self.assertEqual(entry['component'], 'comp') + self.assertEqual(entry['vendor'], 'vendor') + self.assertEqual(entry['version'], '1.0.0') + self.assertEqual(entry['latest'], '1.0.0') + self.assertEqual(entry['url'], 'https://github.com/vendor/comp') + self.assertEqual(entry['file'], 'src/file.py') + self.assertEqual(entry['file_hash'], 'abc123') + self.assertEqual(entry['file_url'], 'https://api.example.com/file_contents/abc123') + self.assertEqual(entry['source_hash'], 'abc123') + self.assertEqual(entry['url_hash'], '') + self.assertEqual(entry['release_date'], '') + self.assertEqual(entry['licenses'], [{'name': 'MIT'}]) + self.assertEqual(entry['lines'], 'all') + self.assertEqual(entry['oss_lines'], 'all') + self.assertEqual(entry['status'], 'pending') + + @patch('scanoss.scanners.scanner_hfh.purl2url') + def test_empty_purl(self, mock_purl2url): + component = {'purl': '', 'name': 'comp', 'vendor': 'vendor'} + best_version = {'version': '1.0.0', 'licenses': []} + + entry = ScannerHFHPresenter._build_file_match_entry( + component, best_version, 'file.py', 'hash', 'https://api.example.com' + ) + + self.assertEqual(entry['purl'], ['']) + self.assertEqual(entry['url'], '') + mock_purl2url.get_repo_url.assert_not_called() + + @patch('scanoss.scanners.scanner_hfh.purl2url') + def test_missing_fields_use_defaults(self, mock_purl2url): + mock_purl2url.get_repo_url.return_value = '' + component = {} + best_version = {} + + entry = ScannerHFHPresenter._build_file_match_entry( + component, best_version, 'file.py', 'hash', 'https://api.example.com' + ) + + self.assertEqual(entry['purl'], ['']) + self.assertEqual(entry['component'], '') + self.assertEqual(entry['vendor'], '') + self.assertEqual(entry['version'], '') + self.assertEqual(entry['licenses'], []) + + @patch('scanoss.scanners.scanner_hfh.purl2url') + def test_purl2url_returns_none(self, mock_purl2url): + mock_purl2url.get_repo_url.return_value = None + component = {'purl': 'pkg:github/vendor/comp', 'name': 'comp', 'vendor': 'vendor'} + best_version = {'version': '1.0.0', 'licenses': []} + + entry = ScannerHFHPresenter._build_file_match_entry( + component, best_version, 'file.py', 'hash', 'https://api.example.com' + ) + + # url should fallback to '' when purl2url returns None/falsy + self.assertEqual(entry['url'], '') + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 0007586a46388a848f40facf1f8428f6e4272cf6 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Thu, 26 Mar 2026 10:24:50 -0300 Subject: [PATCH 483/489] chore(hfh):SP-4188 include license info into hfh results --- CHANGELOG.md | 3 + src/scanoss/scanners/scanner_hfh.py | 136 +++++++++++++++++++++++++--- tests/test_scanner_hfh.py | 78 +++++++++++++++- 3 files changed, 201 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8704b501..ea33adcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `--format raw` option to `folder-scan` command to export HFH results in snippet-scanner JSON format - Expands directory-level HFH results into per-file entries keyed by relative file path - Assigns each file to the most specific matching `path_id` (deepest directory match wins) +- Added license decoration to folder hash scan results via dependency service + - Each component version in HFH results is now decorated with license information + - CycloneDX output uses pre-decorated licenses instead of making a separate dependency API call ## [1.50.1] - 2026-03-23 ### Fixed diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index b5c430e9..ee35a20b 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -116,17 +116,108 @@ def __init__( # noqa: PLR0913 def _execute_grpc_scan(self, hfh_request: Dict) -> None: """ - Execute folder hash scan. + Execute folder hash scan and decorate results with license information. Args: hfh_request: Request dictionary for the gRPC call """ try: self.scan_results = self.client.folder_hash_scan(hfh_request, self.use_grpc) + self._decorate_with_licenses() except Exception as e: self.base.print_stderr(f'Error during folder hash scan: {e}') self.scan_results = None + def _decorate_with_licenses(self) -> None: + """ + Decorate each component version in scan results with license information + by calling the dependency service. + """ + if not self.scan_results or not self.client: + return + results = self.scan_results.get('results', []) + if not results: + return + + dep_files = self._collect_dep_files(results) + if not dep_files: + return + + try: + decorated = self.client.get_dependencies({'files': dep_files}) + except Exception as e: + self.base.print_stderr(f'Warning: Failed to fetch license data: {e}') + return + + if not decorated or 'files' not in decorated: + return + + license_map = self._build_license_map(decorated) + self._inject_licenses(results, license_map) + + @staticmethod + def _collect_dep_files(results: List[Dict]) -> List[Dict]: + """Collect dependency file entries for all component versions in the results.""" + dep_files = [] + for result in results: + path_id = result.get('path_id', '') + for component in result.get('components', []): + purl = component.get('purl', '') + if not purl: + continue + for version_entry in component.get('versions', []): + version = version_entry.get('version', '') + if not version: + continue + dep_files.append({ + 'file': path_id, + 'purls': [{'purl': purl, 'requirement': version}], + }) + return dep_files + + @staticmethod + def _build_license_map(decorated: Dict) -> Dict[str, List]: + """Build a purl@requirement -> licenses lookup from the dependency service response. + + Args: + decorated (Dict): The response from the dependency service containing + decorated files with license information. + + Returns: + Dict[str, List]: A mapping of 'purl@requirement' keys to their + corresponding list of license dictionaries. + """ + license_map = {} + for dep_file in decorated.get('files', []): + for dep in dep_file.get('dependencies', []): + dep_purl = dep.get('purl', '') + # Use 'requirement' instead of 'version' as the key because the service + # may resolve a different version, but the requirement always matches what was sent. + dep_requirement = dep.get('requirement', '') + licenses = dep.get('licenses', []) + if dep_purl and licenses: + license_map[f'{dep_purl}@{dep_requirement}'] = licenses + return license_map + + @staticmethod + def _inject_licenses(results: List[Dict], license_map: Dict[str, List]) -> None: + """Inject licenses from the lookup map into each component version entry. + + Args: + results (List[Dict]): The 'results' list from the HFH scan response. + Each result contains components with version entries that will + be mutated in place to include license data. + license_map (Dict[str, List]): A mapping of 'purl@version' keys to + their corresponding list of license dictionaries, as built by + ``_build_license_map``. + """ + for result in results: + for component in result.get('components', []): + purl = component.get('purl', '') + for version_entry in component.get('versions', []): + version = version_entry.get('version', '') + version_entry['licenses'] = license_map.get(f'{purl}@{version}', []) + def scan(self) -> Optional[Dict]: """ Scan the provided directory using the folder hashing algorithm. @@ -215,30 +306,34 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 if not best_match_component.get('versions'): self.base.print_stderr('ERROR: No versions found for best match component') return '' - best_match_version = best_match_component['versions'][0] purl = best_match_component['purl'] + version = best_match_version['version'] + licenses = best_match_version.get('licenses', []) - get_dependencies_json_request = { - 'files': [ + # Build scan_results from already-decorated HFH data + scan_results = { + f'{best_match_component["name"]}:{version}': [ { - 'file': f'{best_match_component["name"]}:{best_match_version["version"]}', - 'purls': [{'purl': purl, 'requirement': best_match_version['version']}], + 'id': 'dependency', + 'dependencies': [ + { + 'purl': purl, + 'component': best_match_component.get('name', ''), + 'version': version, + 'licenses': licenses, + } + ], } ] } get_vulnerabilities_json_request = { - 'components': [{'purl': purl, 'requirement': best_match_version['version']}], + 'components': [{'purl': purl, 'requirement': version}], } - - decorated_scan_results = self.scanner.client.get_dependencies(get_dependencies_json_request) vulnerabilities = self.scanner.client.get_vulnerabilities_json(get_vulnerabilities_json_request) cdx = CycloneDx(self.base.debug) - scan_results = {} - for f in decorated_scan_results['files']: - scan_results[f['file']] = [f] success, cdx_output = cdx.produce_from_json(scan_results) if not success: error_msg = 'ERROR: Failed to produce CycloneDX output' @@ -250,7 +345,7 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 return json.dumps(cdx_output, indent=2) except Exception as e: - self.base.print_stderr(f'ERROR: Failed to get license information: {e}') + self.base.print_stderr(f'ERROR: Failed to produce CycloneDX output: {e}') return None def _format_spdxlite_output(self) -> str: @@ -411,6 +506,19 @@ def _build_file_match_entry( """ purl = component.get('purl', '') version = best_version.get('version', '') + licenses = [ + { + 'name': lic.get('spdx_id') or lic.get('name', ''), + 'patent_hints': '', + 'copyleft': '', + 'checklist_url': '', + 'incompatible_with': '', + 'osadl_updated': '', + 'source': 'component_declared', + 'url': f"https://spdx.org/licenses/{lic['spdx_id']}.html" if lic.get('spdx_id') else '', + } + for lic in best_version.get('licenses', []) + ] url = purl2url.get_repo_url(purl) if purl else '' return { @@ -428,7 +536,7 @@ def _build_file_match_entry( 'source_hash': file_hash, 'url_hash': '', 'release_date': '', - 'licenses': [], + 'licenses': licenses, 'lines': 'all', 'oss_lines': 'all', 'status': 'pending', diff --git a/tests/test_scanner_hfh.py b/tests/test_scanner_hfh.py index b5d2bb81..4f2bbf8a 100644 --- a/tests/test_scanner_hfh.py +++ b/tests/test_scanner_hfh.py @@ -212,7 +212,13 @@ class TestBuildFileMatchEntry(unittest.TestCase): def test_basic_entry(self, mock_purl2url): mock_purl2url.get_repo_url.return_value = 'https://github.com/vendor/comp' component = {'purl': 'pkg:github/vendor/comp', 'name': 'comp', 'vendor': 'vendor'} - best_version = {'version': '1.0.0', 'licenses': [{'name': 'MIT'}]} + # HFH API license format + best_version = { + 'version': '1.0.0', + 'licenses': [ + {'name': 'MIT License', 'spdx_id': 'MIT', 'is_spdx_approved': True, 'url': 'https://spdx.org/licenses/MIT.html'}, + ], + } entry = ScannerHFHPresenter._build_file_match_entry( component, best_version, 'src/file.py', 'abc123', 'https://api.example.com' @@ -232,7 +238,17 @@ def test_basic_entry(self, mock_purl2url): self.assertEqual(entry['source_hash'], 'abc123') self.assertEqual(entry['url_hash'], '') self.assertEqual(entry['release_date'], '') - self.assertEqual(entry['licenses'], [{'name': 'MIT'}]) + # License should be transformed from HFH format to snippet-scanner format + self.assertEqual(len(entry['licenses']), 1) + lic = entry['licenses'][0] + self.assertEqual(lic['name'], 'MIT') + self.assertEqual(lic['source'], 'component_declared') + self.assertEqual(lic['url'], 'https://spdx.org/licenses/MIT.html') + self.assertEqual(lic['patent_hints'], '') + self.assertEqual(lic['copyleft'], '') + self.assertEqual(lic['checklist_url'], '') + self.assertEqual(lic['incompatible_with'], '') + self.assertEqual(lic['osadl_updated'], '') self.assertEqual(entry['lines'], 'all') self.assertEqual(entry['oss_lines'], 'all') self.assertEqual(entry['status'], 'pending') @@ -266,6 +282,64 @@ def test_missing_fields_use_defaults(self, mock_purl2url): self.assertEqual(entry['version'], '') self.assertEqual(entry['licenses'], []) + @patch('scanoss.scanners.scanner_hfh.purl2url') + def test_license_uses_spdx_id_as_name(self, mock_purl2url): + mock_purl2url.get_repo_url.return_value = '' + component = {'purl': 'pkg:github/v/c', 'name': 'c', 'vendor': 'v'} + best_version = { + 'version': '1.0', + 'licenses': [ + {'name': 'GNU General Public License v2.0 only', 'spdx_id': 'GPL-2.0-only', 'is_spdx_approved': True, 'url': 'https://spdx.org/licenses/GPL-2.0-only.html'}, + ], + } + + entry = ScannerHFHPresenter._build_file_match_entry( + component, best_version, 'file.py', 'hash', 'https://api.example.com' + ) + + lic = entry['licenses'][0] + self.assertEqual(lic['name'], 'GPL-2.0-only') + self.assertEqual(lic['url'], 'https://spdx.org/licenses/GPL-2.0-only.html') + + @patch('scanoss.scanners.scanner_hfh.purl2url') + def test_license_without_spdx_id_falls_back_to_name(self, mock_purl2url): + mock_purl2url.get_repo_url.return_value = '' + component = {'purl': 'pkg:github/v/c', 'name': 'c', 'vendor': 'v'} + best_version = { + 'version': '1.0', + 'licenses': [{'name': 'Some Custom License'}], + } + + entry = ScannerHFHPresenter._build_file_match_entry( + component, best_version, 'file.py', 'hash', 'https://api.example.com' + ) + + lic = entry['licenses'][0] + self.assertEqual(lic['name'], 'Some Custom License') + self.assertEqual(lic['url'], '') + + @patch('scanoss.scanners.scanner_hfh.purl2url') + def test_multiple_licenses_transformed(self, mock_purl2url): + mock_purl2url.get_repo_url.return_value = '' + component = {'purl': 'pkg:github/v/c', 'name': 'c', 'vendor': 'v'} + best_version = { + 'version': '1.0', + 'licenses': [ + {'name': 'MIT License', 'spdx_id': 'MIT', 'is_spdx_approved': True, 'url': 'https://spdx.org/licenses/MIT.html'}, + {'name': 'Apache License 2.0', 'spdx_id': 'Apache-2.0', 'is_spdx_approved': True, 'url': 'https://spdx.org/licenses/Apache-2.0.html'}, + ], + } + + entry = ScannerHFHPresenter._build_file_match_entry( + component, best_version, 'file.py', 'hash', 'https://api.example.com' + ) + + self.assertEqual(len(entry['licenses']), 2) + self.assertEqual(entry['licenses'][0]['name'], 'MIT') + self.assertEqual(entry['licenses'][0]['url'], 'https://spdx.org/licenses/MIT.html') + self.assertEqual(entry['licenses'][1]['name'], 'Apache-2.0') + self.assertEqual(entry['licenses'][1]['url'], 'https://spdx.org/licenses/Apache-2.0.html') + @patch('scanoss.scanners.scanner_hfh.purl2url') def test_purl2url_returns_none(self, mock_purl2url): mock_purl2url.get_repo_url.return_value = None From 0af5c2d8fae82d99ffd593a4e50895c33e947264 Mon Sep 17 00:00:00 2001 From: Agustin Groh Date: Thu, 26 Mar 2026 13:10:22 -0300 Subject: [PATCH 484/489] chore(version): upgrade version to v1.51.0 --- CHANGELOG.md | 4 ++++ src/scanoss/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea33adcb..f53e4860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [1.51.0] - 2026-03-26 ### Added - Added `--format raw` option to `folder-scan` command to export HFH results in snippet-scanner JSON format - Expands directory-level HFH results into per-file entries keyed by relative file path @@ -859,3 +861,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.49.0]: https://github.com/scanoss/scanoss.py/compare/v1.48.0...v1.49.0 [1.49.1]: https://github.com/scanoss/scanoss.py/compare/v1.49.0...v1.49.1 [1.50.0]: https://github.com/scanoss/scanoss.py/compare/v1.49.1...v1.50.0 +[1.50.1]: https://github.com/scanoss/scanoss.py/compare/v1.50.0...v1.50.1 +[1.51.0]: https://github.com/scanoss/scanoss.py/compare/v1.50.1...v1.51.0 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index d68def05..4aaf545d 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.50.1' +__version__ = '1.51.0' From faa765629fd689f314d2bda43088cfd9b04d0736 Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Wed, 1 Apr 2026 08:20:53 +0000 Subject: [PATCH 485/489] fix(cyclonedx): resolve missing vulnerabilities in folder-scan output The append_vulnerabilities method was reading the wrong key from the vulnerability API response ('purls' instead of 'components'), causing vulnerabilities to be silently dropped from CycloneDX output. --- CHANGELOG.md | 4 ++++ src/scanoss/__init__.py | 2 +- src/scanoss/cyclonedx.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f53e4860..586f6d98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.51.1] - 2026-04-01 +### Fixed +- Fixed vulnerabilities not appearing in CycloneDX output for folder-scan (`fs`) command + ## [1.51.0] - 2026-03-26 ### Added - Added `--format raw` option to `folder-scan` command to export HFH results in snippet-scanner JSON format diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 4aaf545d..f599b249 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.51.0' +__version__ = '1.51.1' diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index e1012605..909d33d3 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -352,7 +352,7 @@ def append_vulnerabilities(self, cdx_dict: dict, vulnerabilities_data: dict, pur cdx_dict['vulnerabilities'] = [] # Extract vulnerabilities from the response - vulns_list = vulnerabilities_data.get('purls', []) + vulns_list = vulnerabilities_data.get('components', []) if not vulns_list: return cdx_dict From 84f0ffe2c76892b19f756fee30c19fa3a1ff31ff Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Wed, 1 Apr 2026 08:34:55 +0000 Subject: [PATCH 486/489] docs(changelog): add missing compare link for v1.51.1 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 586f6d98..0cf23a87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -867,3 +867,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.50.0]: https://github.com/scanoss/scanoss.py/compare/v1.49.1...v1.50.0 [1.50.1]: https://github.com/scanoss/scanoss.py/compare/v1.50.0...v1.50.1 [1.51.0]: https://github.com/scanoss/scanoss.py/compare/v1.50.1...v1.51.0 +[1.51.1]: https://github.com/scanoss/scanoss.py/compare/v1.51.0...v1.51.1 From 72c17ac502706e2631ea721c1aa2bb8cff8638eb Mon Sep 17 00:00:00 2001 From: Agustin Isasmendi Date: Wed, 1 Apr 2026 09:04:16 +0000 Subject: [PATCH 487/489] fix(cyclonedx): suppress spurious stdout print during folder-scan output --- CHANGELOG.md | 1 + src/scanoss/cyclonedx.py | 20 +++++++++++--------- src/scanoss/scanners/scanner_hfh.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cf23a87..a385c073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.51.1] - 2026-04-01 ### Fixed - Fixed vulnerabilities not appearing in CycloneDX output for folder-scan (`fs`) command +- Fixed CycloneDX output without vulnerabilities being printed to stdout when using `--output` with folder-scan ## [1.51.0] - 2026-03-26 ### Added diff --git a/src/scanoss/cyclonedx.py b/src/scanoss/cyclonedx.py index 909d33d3..ddf10912 100644 --- a/src/scanoss/cyclonedx.py +++ b/src/scanoss/cyclonedx.py @@ -180,13 +180,14 @@ def produce_from_file(self, json_file: str, output_file: str = None) -> bool: success = self.produce_from_str(f.read(), output_file) return success - def produce_from_json(self, data: dict, output_file: str = None) -> tuple[bool, dict]: # noqa: PLR0912 + def produce_from_json(self, data: dict, output_file: str = None, print_output: bool = True) -> tuple[bool, dict]: # noqa: PLR0912 """ Produce the CycloneDX output from the raw scan results input data Args: data (dict): JSON object output_file (str, optional): Output file (optional). Defaults to None. + print_output (bool, optional): Print/write output. Defaults to True. Returns: bool: True if successful, False otherwise @@ -273,14 +274,15 @@ def produce_from_json(self, data: dict, output_file: str = None) -> tuple[bool, data['vulnerabilities'].append(vd) # End for loop - file = sys.stdout - if not output_file and self.output_file: - output_file = self.output_file - if output_file: - file = open(output_file, 'w') - print(json.dumps(data, indent=2), file=file) - if output_file: - file.close() + if print_output: + file = sys.stdout + if not output_file and self.output_file: + output_file = self.output_file + if output_file: + file = open(output_file, 'w') + print(json.dumps(data, indent=2), file=file) + if output_file: + file.close() return True, data diff --git a/src/scanoss/scanners/scanner_hfh.py b/src/scanoss/scanners/scanner_hfh.py index ee35a20b..06deca97 100644 --- a/src/scanoss/scanners/scanner_hfh.py +++ b/src/scanoss/scanners/scanner_hfh.py @@ -334,7 +334,7 @@ def _format_cyclonedx_output(self) -> str: # noqa: PLR0911 vulnerabilities = self.scanner.client.get_vulnerabilities_json(get_vulnerabilities_json_request) cdx = CycloneDx(self.base.debug) - success, cdx_output = cdx.produce_from_json(scan_results) + success, cdx_output = cdx.produce_from_json(scan_results, print_output=False) if not success: error_msg = 'ERROR: Failed to produce CycloneDX output' self.base.print_stderr(error_msg) From bcd043f28bc227b2c722625017e5ab785ec487e3 Mon Sep 17 00:00:00 2001 From: scanoss-qg <78024084+scanoss-qg@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:18:25 -0300 Subject: [PATCH 488/489] feat(components): Support for querying Development status * feat(components): Support for querying Development status for a component. * fix(components): Fix typos - Renamed aliases based on PR comments * Updated changelog * This closes SP4197 --- CHANGELOG.md | 8 ++++++ CLIENT_HELP.md | 31 ++++++++++++++++++++++- src/scanoss/__init__.py | 2 +- src/scanoss/cli.py | 52 ++++++++++++++++++++++++++++++++++++++ src/scanoss/components.py | 33 ++++++++++++++++++++++++ src/scanoss/scanossgrpc.py | 24 ++++++++++++++++++ 6 files changed, 148 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a385c073..69d150e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] + +## [1.52.0] - 2026-04-09 +### Added +- Added `status` subcommand query to `component` command to retrieve development life-cycle status: + - Component and version-specific for a single component + - Component and version-specific for a list of components + ## [1.51.1] - 2026-04-01 ### Fixed - Fixed vulnerabilities not appearing in CycloneDX output for folder-scan (`fs`) command @@ -869,3 +876,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.50.1]: https://github.com/scanoss/scanoss.py/compare/v1.50.0...v1.50.1 [1.51.0]: https://github.com/scanoss/scanoss.py/compare/v1.50.1...v1.51.0 [1.51.1]: https://github.com/scanoss/scanoss.py/compare/v1.51.0...v1.51.1 +[1.52.0]: https://github.com/scanoss/scanoss.py/compare/v1.51.1...v1.52.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index 23da2c77..e23a44a7 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -423,6 +423,8 @@ The `component` command has a suite of sub-commands designed to operate on OSS c * Version Details (`versions`) * Cryptography (`crypto`) * Provenance (`provenance`) +* Licenses (`licenses`) +* Status (`status`) For the latest list of sub-commands, please run: ```bash @@ -518,14 +520,38 @@ The licenses command also supports CycloneDX (CDX) input files. You can provide scanoss-py comp licenses -i cyclonedx-sbom.json -o component-licenses.json ``` +#### Component Status +The following command provides the capability to search the SCANOSS KB for development status information of Open Source components: +```bash +scanoss-py comp status -p "pkg:npm/react@17.0.2" +``` +It is possible to supply multiple PURLs by repeating the `-p pkg` option, or providing a purl input file `-i purl-input.json` ([for example](tests/data/purl-input.json)): +```bash +scanoss-py comp status -i purl-input.json -o component-status.json +``` + +The status command also supports CycloneDX (CDX) input files. You can provide a CycloneDX SBOM file and retrieve status information for all components: +```bash +scanoss-py comp status -i cyclonedx-sbom.json -o component-status.json +``` + +The component status provides information about: +- **Component status**: Overall status of the component (active, inactive, deprecated) +- **Repository status**: Current status of the component's repository +- **First indexed date**: When the component was first indexed in SCANOSS KB +- **Last indexed date**: Most recent indexing date +- **Version status**: Status specific to the requested version +- **Indexed date**: When the specific version was indexed + ### CDX Input Support for Component Commands Several component commands now support CycloneDX (CDX) input files. This allows you to analyze components from existing SBOM files: **Supported commands with CDX input:** - `comp vulns` - Analyze vulnerabilities from CDX file -- `comp licenses` - Retrieve licenses from CDX file +- `comp licenses` - Retrieve licenses from CDX file - `comp crypto` - Detect cryptographic algorithms from CDX file - `comp semgrep` - Find semgrep issues from CDX file +- `comp status` - Retrieve development status from CDX file **Example using CDX input:** ```bash @@ -535,6 +561,9 @@ scanoss-py comp vulns -i sbom.cdx.json -o vulnerabilities.json # Get licenses for all components in a CycloneDX SBOM scanoss-py comp licenses -i sbom.cdx.json -o licenses.json +# Get status information for all components in a CycloneDX SBOM +scanoss-py comp status -i sbom.cdx.json -o status.json + # Detect cryptographic usage from CDX scanoss-py comp crypto -i sbom.cdx.json -o crypto-findings.json ``` diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index f599b249..82c1bc30 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.51.1' +__version__ = '1.52.0' diff --git a/src/scanoss/cli.py b/src/scanoss/cli.py index 29282f8d..af5f36f9 100644 --- a/src/scanoss/cli.py +++ b/src/scanoss/cli.py @@ -420,6 +420,15 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_versions.add_argument('--limit', '-l', type=int, help='Generic component search') c_versions.set_defaults(func=comp_versions) + # Component Sub-command: component status + c_status = comp_sub.add_parser( + 'status', + aliases=['sts','st'], + description=f'Show Component Status details: {__version__}', + help='Retrieve development status for the given components', + ) + c_status.set_defaults(func=comp_status) + # Sub-command: crypto p_crypto = subparsers.add_parser( 'crypto', @@ -478,6 +487,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_hints, p_crypto_versions_in_range, c_licenses, + c_status, ]: p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.') p.add_argument('--input', '-i', type=str, help='Input file name') @@ -493,6 +503,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_hints, p_crypto_versions_in_range, c_licenses, + c_status, ]: p.add_argument( '--timeout', @@ -510,6 +521,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_semgrep, c_provenance, c_licenses, + c_status, ]: p.add_argument( '--apiurl', type=str, help='SCANOSS API base URL (optional - default: https://api.osskb.org)' @@ -1090,6 +1102,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_hints, p_crypto_versions_in_range, c_licenses, + c_status, p_copy, ]: p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).') @@ -1178,6 +1191,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_hints, p_crypto_versions_in_range, c_licenses, + c_status, ]: p.add_argument( '--key', '-k', type=str, help='SCANOSS API Key token (optional - not required for default OSSKB URL)' @@ -1215,6 +1229,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_hints, p_crypto_versions_in_range, c_licenses, + c_status, ]: p.add_argument( '--api2url', type=str, @@ -1261,6 +1276,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 c_search, c_versions, c_licenses, + c_status, p_folder_scan, ]: p.add_argument( @@ -1305,6 +1321,7 @@ def setup_args() -> None: # noqa: PLR0912, PLR0915 p_crypto_hints, p_crypto_versions_in_range, c_licenses, + c_status, e_dt, p_copy, ]: @@ -2652,6 +2669,41 @@ def comp_licenses(parser, args): sys.exit(1) +def comp_status(parser, args): + """ + Run the "component status" sub-command + Parameters + ---------- + parser: ArgumentParser + command line parser object + args: Namespace + Parsed arguments + """ + if (not args.purl and not args.input) or (args.purl and args.input): + print_stderr('ERROR: Please specify an input file or purl to decorate (--purl or --input)') + parser.parse_args([args.subparser, args.subparsercmd, '-h']) + sys.exit(1) + if args.ca_cert and not os.path.exists(args.ca_cert): + print_stderr(f'ERROR: Certificate file does not exist: {args.ca_cert}.') + sys.exit(1) + pac_file = get_pac_file(args.pac) + comps = Components( + debug=args.debug, + trace=args.trace, + quiet=args.quiet, + grpc_url=args.apiurl, # Legacy param name; accepts the REST API base URL. TODO: rename to url + api_key=args.key, + ca_cert=args.ca_cert, + proxy=args.proxy, + pac=pac_file, + timeout=args.timeout, + req_headers=process_req_headers(args.header), + ignore_cert_errors=args.ignore_cert_errors, + ) + if not comps.get_status(args.input, args.purl, args.output): + sys.exit(1) + + def results(parser, args): """ Run the "results" sub-command diff --git a/src/scanoss/components.py b/src/scanoss/components.py index 6498c1a2..34cf2212 100644 --- a/src/scanoss/components.py +++ b/src/scanoss/components.py @@ -395,3 +395,36 @@ def get_licenses(self, json_file: str = None, purls: [] = None, output_file: str self.print_msg(f'Results written to: {output_file}') self._close_file(output_file, file) return success + + def get_status(self, json_file: str = None, purls: [] = None, output_file: str = None) -> bool: + """ + Retrieve the development status details for the supplied PURLs + + Args: + json_file (str, optional): Input JSON file. Defaults to None. + purls (None, optional): PURLs to retrieve status details for. Defaults to None. + output_file (str, optional): Output file. Defaults to None. + + Returns: + bool: True on success, False otherwise + """ + success = False + + purls_request = self.load_purls(json_file, purls) + if not purls_request: + return False + file = self._open_file_or_sdtout(output_file) + if file is None: + return False + + # Use ComponentBatchRequest format for the status api + component_batch_request = {'components': purls_request.get('purls')} + self.print_msg('Sending PURLs to Component Status API...') + response = self.grpc_api.get_component_status(component_batch_request, use_grpc=self.use_grpc) + if response: + print(json.dumps(response, indent=2, sort_keys=True), file=file) + success = True + if output_file: + self.print_msg(f'Results written to: {output_file}') + self._close_file(output_file, file) + return success diff --git a/src/scanoss/scanossgrpc.py b/src/scanoss/scanossgrpc.py index 490ae72a..1eab55e0 100644 --- a/src/scanoss/scanossgrpc.py +++ b/src/scanoss/scanossgrpc.py @@ -95,6 +95,7 @@ }, 'components.SearchComponents': {'path': '/components/search', 'method': 'GET'}, 'components.GetComponentVersions': {'path': '/components/versions', 'method': 'GET'}, + 'components.GetComponentsStatus': {'path': '/components/status/components', 'method': 'POST'}, 'geoprovenance.GetCountryContributorsByComponents': { 'path': '/geoprovenance/countries/components', 'method': 'POST', @@ -765,6 +766,29 @@ def get_licenses(self, request: Dict, use_grpc: Optional[bool] = None) -> Option use_grpc=use_grpc, ) + def get_component_status(self, request: Dict, use_grpc: Optional[bool] = None) -> Optional[Dict]: + """ + Client function to call the API for Components GetComponentsStatus + Only REST API is supported for this endpoint (gRPC not yet available) + + Args: + request (Dict): ComponentsRequest + Returns: + Optional[Dict]: ComponentsStatusResponse, or None if the request was not successful + """ + # Force REST API since gRPC is not available for this endpoint + if use_grpc: + self.print_stderr('WARNING: gRPC is not supported for component status. Using REST API instead.') + + return self._call_api( + 'components.GetComponentsStatus', + None, # No gRPC method available + request, + ComponentsRequest, + 'Sending data for component status retrieval (rqId: {rqId})...', + use_grpc=False, # Force REST + ) + def load_generic_headers(self, url: Optional[str] = None): """ Adds custom headers from req_headers to metadata. From 33df27711c3ae77ef945128882a0fc9e77e87515 Mon Sep 17 00:00:00 2001 From: Matias Daloia <66310421+matiasdaloia@users.noreply.github.com> Date: Wed, 15 Apr 2026 09:00:59 +0200 Subject: [PATCH 489/489] [SP-4275] feat: extract and add cpe's to spdx output (#204) * [SP-4275] feat: extract and add cpe's to spdx output * [SP-4275] chore: normalize cpe to lowercase --- .gitignore | 1 + CHANGELOG.md | 8 ++ src/scanoss/__init__.py | 2 +- src/scanoss/spdxlite.py | 92 +++++++++++++++++++++-- tests/test_spdxlite.py | 163 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 257 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index da345dfc..8fc669fd 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ docs/build .DS_Store !scanoss.json examples/output/ +!spdx-*.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 69d150e8..27c34a75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.52.1] - 2026-04-14 +### Fixed +- Fixed CPE identifiers missing from SPDX Lite output (`--format spdxlite`) + - CPEs are now emitted as SPDX 2.2 `externalRefs` with `referenceCategory: SECURITY` + - CPE 2.3 strings use `referenceType: cpe23Type`; legacy `cpe:/...` and `cpe:2.2:...` use `cpe22Type` + - Multiple CPEs per component are preserved and deduplicated + ## [1.52.0] - 2026-04-09 ### Added - Added `status` subcommand query to `component` command to retrieve development life-cycle status: @@ -877,3 +884,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [1.51.0]: https://github.com/scanoss/scanoss.py/compare/v1.50.1...v1.51.0 [1.51.1]: https://github.com/scanoss/scanoss.py/compare/v1.51.0...v1.51.1 [1.52.0]: https://github.com/scanoss/scanoss.py/compare/v1.51.1...v1.52.0 +[1.52.0]: https://github.com/scanoss/scanoss.py/compare/v1.51.1...v1.52.1 diff --git a/src/scanoss/__init__.py b/src/scanoss/__init__.py index 82c1bc30..7fd460f3 100644 --- a/src/scanoss/__init__.py +++ b/src/scanoss/__init__.py @@ -22,4 +22,4 @@ THE SOFTWARE. """ -__version__ = '1.52.0' +__version__ = '1.52.1' diff --git a/src/scanoss/spdxlite.py b/src/scanoss/spdxlite.py index 3e13af89..89bd70ef 100644 --- a/src/scanoss/spdxlite.py +++ b/src/scanoss/spdxlite.py @@ -219,8 +219,41 @@ def _create_file_summary(self, entry: dict) -> dict: for field in fields: summary[field] = entry.get(field) summary['licenses'] = self._process_licenses(entry.get('licenses')) + summary['cpes'] = self._extract_cpes(entry.get('vulnerabilities')) return summary + def _extract_cpes(self, vulnerabilities: list) -> list: + """ + Extract CPE identifiers from a file entry's vulnerabilities array. + + Raw scan results deliver CPEs embedded as vulnerability IDs prefixed with "CPE:" + (case-insensitive). Everything else in the array is a real vulnerability record + (CVE/GHSA) and must be ignored here. + + Args: + vulnerabilities (list): The 'vulnerabilities' list from a file match entry. + May be None or empty. + + Returns: + list: Deduplicated list of CPE strings in source order (e.g. + ['cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*']). + Returns an empty list when there are no CPE entries. + """ + if not vulnerabilities: + return [] + cpes = [] + seen = set() + for vuln in vulnerabilities: + vuln_id = vuln.get('ID') or vuln.get('id') or '' + if not vuln_id.upper().startswith('CPE:'): + continue + normalized = vuln_id.upper() + if normalized in seen: + continue + seen.add(normalized) + cpes.append(vuln_id) + return cpes + def _process_licenses(self, licenses: list) -> list: """ Process license information and remove duplicates. @@ -426,6 +459,15 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict: purl_ver = f'{purl}@{comp_ver}' purl_hash = hashlib.md5(purl_ver.encode('utf-8')).hexdigest() + external_refs = [ + { + 'referenceCategory': 'PACKAGE-MANAGER', + 'referenceLocator': PackageURL.from_string(purl_ver).to_string(), + 'referenceType': 'purl' + } + ] + external_refs.extend(self._create_cpe_external_refs(comp.get('cpes', []))) + return { 'name': comp.get('component'), 'SPDXID': f'SPDXRef-{purl_hash}', @@ -437,13 +479,7 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict: 'filesAnalyzed': False, 'copyrightText': 'NOASSERTION', 'supplier': f'Organization: {comp.get("vendor", "NOASSERTION")}', - 'externalRefs': [ - { - 'referenceCategory': 'PACKAGE-MANAGER', - 'referenceLocator': PackageURL.from_string(purl_ver).to_string(), - 'referenceType': 'purl' - } - ], + 'externalRefs': external_refs, 'checksums': [ { 'algorithm': 'MD5', @@ -452,6 +488,48 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict: ], } + def _create_cpe_external_refs(self, cpes: list) -> list: + """ + Build SPDX externalRefs entries for a component's CPE identifiers. + + SPDX 2.2 models CPEs under the SECURITY reference category. Each CPE string + must be emitted as its own externalRef dict with the shape: + + { + 'referenceCategory': 'SECURITY', + 'referenceType': 'cpe23Type' | 'cpe22Type', + 'referenceLocator': '', + } + + Args: + cpes (list): CPE strings extracted from the raw scan results. The list is + already deduplicated by `_extract_cpes`. Values look like + 'cpe:2.3:a:vendor:product:version:...' (CPE 2.3) or + 'cpe:/a:vendor:product:version' (legacy CPE 2.2). May be empty. + + Returns: + list: A list of SPDX externalRef dicts ready to be appended to a package's + `externalRefs`. Return an empty list when `cpes` is empty. + """ + if not cpes: + return [] + refs = [] + for cpe in cpes: + normalized = cpe.lower() + if normalized.startswith('cpe:2.3:'): + ref_type = 'cpe23Type' + elif normalized.startswith('cpe:/') or normalized.startswith('cpe:2.2:'): + ref_type = 'cpe22Type' + else: + self.print_debug(f'Warning: Unrecognized CPE format, defaulting to cpe23Type: {cpe}') + ref_type = 'cpe23Type' + refs.append({ + 'referenceCategory': 'SECURITY', + 'referenceType': ref_type, + 'referenceLocator': cpe, + }) + return refs + def _process_package_licenses(self, licenses: list, lic_refs: set) -> str: """ Process licenses and return license text formatted for SPDX. diff --git a/tests/test_spdxlite.py b/tests/test_spdxlite.py index 6c7ed3f3..7683a4da 100644 --- a/tests/test_spdxlite.py +++ b/tests/test_spdxlite.py @@ -67,4 +67,165 @@ def testSpdxLite(self): self.assertEqual(len(checksum.get("checksumValue")), md5_length) #Check checksum length value be 32 - os.remove(spdx_lite_output) #Removes tmp spdxlite.json file \ No newline at end of file + os.remove(spdx_lite_output) #Removes tmp spdxlite.json file + + +class SpdxLiteCpeTests(unittest.TestCase): + """ + Exercise CPE extraction and SPDX externalRefs emission. + """ + + @staticmethod + def _build_raw(vulnerabilities, purl='pkg:github/postgres/postgres'): + return { + 'src/main.c': [{ + 'id': 'file', + 'component': 'postgresql', + 'vendor': 'postgresql', + 'version': '17.0', + 'latest': '17.0', + 'url': 'https://www.postgresql.org', + 'url_hash': 'abc123', + 'download_url': 'https://example.com/pg.tar.gz', + 'purl': [purl], + 'licenses': [{'name': 'PostgreSQL', 'source': 'component_declared'}], + 'vulnerabilities': vulnerabilities, + }] + } + + def _run(self, raw): + fd, out_path = tempfile.mkstemp(prefix='spdxlite_cpe_', suffix='.json') + os.close(fd) # SpdxLite re-opens the path itself for writing + try: + spdx = SpdxLite(debug=False, output_file=out_path) + spdx.produce_from_json(raw) + with open(out_path, 'r') as f: + return json.load(f) + finally: + if os.path.exists(out_path): + os.remove(out_path) + + def _security_refs(self, doc): + refs = doc['packages'][0]['externalRefs'] + return [r for r in refs if r['referenceCategory'] == 'SECURITY'] + + def test_cpe23_emits_cpe23Type(self): + cpe = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*' + doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceType'], 'cpe23Type') + self.assertEqual(refs[0]['referenceLocator'], cpe) + + def test_legacy_cpe22_slash_emits_cpe22Type(self): + cpe = 'cpe:/a:postgresql:postgresql:17.0' + doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceType'], 'cpe22Type') + self.assertEqual(refs[0]['referenceLocator'], cpe) + + def test_explicit_cpe22_prefix_emits_cpe22Type(self): + cpe = 'cpe:2.2:a:postgresql:postgresql:17.0' + doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceType'], 'cpe22Type') + + def test_case_insensitive_prefix_detection(self): + cpe = 'CPE:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*' + doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceType'], 'cpe23Type') + self.assertEqual(refs[0]['referenceLocator'], cpe) # casing preserved in locator + + def test_duplicate_cpes_are_deduplicated(self): + cpe = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*' + doc = self._run(self._build_raw([ + {'ID': cpe, 'source': 'nvd'}, + {'ID': cpe, 'source': 'nvd'}, + {'ID': cpe, 'source': 'nvd'}, + ])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + + def test_dedup_is_case_insensitive_and_preserves_first_locator(self): + lower = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*' + upper = 'CPE:2.3:A:POSTGRESQL:POSTGRESQL:17.0:*:*:*:*:*:*:*' + doc = self._run(self._build_raw([ + {'ID': lower, 'source': 'nvd'}, + {'ID': upper, 'source': 'nvd'}, + ])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceLocator'], lower) # first-seen wins + + def test_cve_entries_are_ignored(self): + doc = self._run(self._build_raw([ + {'ID': 'CVE-2024-12345', 'CVE': 'CVE-2024-12345', + 'source': 'nvd', 'severity': 'high'}, + {'ID': 'GHSA-xxxx-yyyy-zzzz', 'source': 'github'}, + ])) + refs = self._security_refs(doc) + self.assertEqual(refs, []) + + def test_mixed_cpe_versions_in_same_component(self): + cpe23 = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*' + cpe22 = 'cpe:/a:postgresql:postgresql:17.0' + doc = self._run(self._build_raw([ + {'ID': cpe23, 'source': 'nvd'}, + {'ID': cpe22, 'source': 'nvd'}, + ])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 2) + types = {r['referenceType']: r['referenceLocator'] for r in refs} + self.assertEqual(types['cpe23Type'], cpe23) + self.assertEqual(types['cpe22Type'], cpe22) + + def test_unknown_cpe_format_falls_back_to_cpe23Type(self): + odd_cpe = 'cpe:weird-format:postgresql:17.0' + doc = self._run(self._build_raw([{'ID': odd_cpe, 'source': 'nvd'}])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceType'], 'cpe23Type') + self.assertEqual(refs[0]['referenceLocator'], odd_cpe) + + def test_no_vulnerabilities_field_produces_no_security_refs(self): + raw = self._build_raw([]) + # Drop the key entirely to simulate entries without a vulnerabilities block + del raw['src/main.c'][0]['vulnerabilities'] + doc = self._run(raw) + self.assertEqual(self._security_refs(doc), []) + # PURL externalRef must still be present + refs = doc['packages'][0]['externalRefs'] + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceType'], 'purl') + + def test_empty_vulnerabilities_list_produces_no_security_refs(self): + doc = self._run(self._build_raw([])) + self.assertEqual(self._security_refs(doc), []) + + def test_dependency_entries_do_not_emit_cpes(self): + raw = { + 'package.json': [{ + 'id': 'dependency', + 'dependencies': [{ + 'purl': 'pkg:npm/left-pad', + 'component': 'left-pad', + 'version': '1.3.0', + 'url': 'https://npmjs.com/package/left-pad', + 'licenses': [{'name': 'MIT', 'source': 'component_declared'}], + }] + }] + } + doc = self._run(raw) + self.assertEqual(self._security_refs(doc), []) + + def test_lowercase_id_key_is_also_supported(self): + cpe = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*' + # Raw scan output has been known to use 'id' (lowercase) occasionally + doc = self._run(self._build_raw([{'id': cpe, 'source': 'nvd'}])) + refs = self._security_refs(doc) + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]['referenceType'], 'cpe23Type') \ No newline at end of file