Create test for minecode tasks #362

    * Add new envvar PURLDB_ASYNC

Signed-off-by: Jono Yang <jyang@nexb.com>

minecode/api.py

@@ -218,7 +218,7 @@ def update_status(self, request, *args, **kwargs):
             with open(scan_summary_location, 'wb') as f:
                 f.write(scan_summary_file.read())
 
-            _ = scannable_uri.process_scan_results(
+            scannable_uri.process_scan_results(
                 scan_results_location=scan_results_location,
                 scan_summary_location=scan_summary_location,
                 project_extra_data=project_extra_data

minecode/models.py

@@ -12,6 +12,7 @@
 import logging
 import sys
 
+from django.conf import settings
 from django.db import models
 from django.utils import timezone
 import django_rq
@@ -770,6 +771,16 @@ def process_scan_results(
 
         self.scan_status = self.SCAN_COMPLETED
         self.save()
+
+        if not settings.PURLDB_ASYNC:
+            tasks.process_scan_results(
+                scannable_uri=self,
+                scan_results_location=scan_results_location,
+                scan_summary_location=scan_summary_location,
+                project_extra_data=project_extra_data,
+            )
+            return
+
         job = django_rq.enqueue(
             tasks.process_scan_results,
             scannable_uri=self,

minecode/tasks.py

@@ -22,8 +22,12 @@ def process_scan_results(
     project_extra_data,
 ):
     """
-    Indexes the scan results from `scan_results_file`, `scan_summary_file`, and
-    `project_extra_data` for the Package related to `scannable_uri`
+    Indexes the scan results from `scan_results_location`,
+    `scan_summary_location`, and `project_extra_data` for the Package related to
+    `scannable_uri`
+
+    `scan_results_location` and `scan_summary_location` are deleted after the
+    indexing process has finished.
     """
 
     with open(scan_results_location) as f:
@@ -46,6 +50,7 @@ def process_scan_results(
         scannable_uri.index_error = indexing_errors
     else:
         scannable_uri.scan_status = ScannableURI.SCAN_INDEXED
+
     scannable_uri.wip_date = None
     scannable_uri.save()
 

minecode/tests/test_tasks.py

@@ -0,0 +1,74 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import os
+
+from django.test import TestCase
+from unittest import mock
+
+from minecode.models import ScannableURI
+from packagedb.models import Package
+from minecode.utils_test import JsonBasedTesting
+from minecode import tasks
+
+
+class MinecodeTasksTestCase(JsonBasedTesting, TestCase):
+    test_data_dir = os.path.join(os.path.dirname(__file__), 'testfiles')
+
+    def setUp(self):
+        self.package1 = Package.objects.create(
+            download_url='https://test-url.com/package1.tar.gz',
+            type='type1',
+            name='name1',
+            version='1.0',
+        )
+        self.scannable_uri1 = ScannableURI.objects.create(
+            uri='https://test-url.com/package1.tar.gz',
+            package=self.package1
+        )
+        self.project_extra_data1 = {
+            'md5': 'md5',
+            'sha1': 'sha1',
+            'sha256': 'sha256',
+            'sha512': 'sha512',
+            'size': 100,
+        }
+
+    @mock.patch('os.remove')
+    def test_minecode_tasks_process_scan_results(self, mock_delete):
+        mock_delete.side_effect = [None, None]
+
+        self.assertFalse(self.package1.md5)
+        self.assertFalse(self.package1.sha1)
+        self.assertFalse(self.package1.sha256)
+        self.assertFalse(self.package1.sha512)
+        self.assertFalse(self.package1.size)
+        self.assertFalse(self.package1.declared_license_expression)
+        self.assertFalse(self.package1.copyright)
+        self.assertEquals(0, self.package1.resources.count())
+        scan_file_location = self.get_test_loc('scancodeio/get_scan_data.json')
+        summary_file_location = self.get_test_loc('scancodeio/scan_summary_response.json')
+        project_extra_data = json.dumps(self.project_extra_data1)
+        tasks.process_scan_results(
+            self.scannable_uri1,
+            scan_results_location=scan_file_location,
+            scan_summary_location=summary_file_location,
+            project_extra_data=project_extra_data,
+        )
+        self.package1.refresh_from_db()
+        self.assertEqual('md5', self.package1.md5)
+        self.assertEqual('sha1', self.package1.sha1)
+        self.assertEqual('sha256', self.package1.sha256)
+        self.assertEqual('sha512', self.package1.sha512)
+        self.assertEqual(100, self.package1.size)
+        self.assertEqual('apache-2.0', self.package1.declared_license_expression)
+        self.assertEqual('Copyright (c) Apache Software Foundation', self.package1.copyright)
+        self.assertFalse(self.scannable_uri1.scan_error)
+        self.assertEqual(64, self.package1.resources.count())

purldb_project/settings.py

@@ -308,6 +308,7 @@
     'VERSION': PURLDB_VERSION,
     'SERVE_INCLUDE_SCHEMA': False,
 }
+
 RQ_QUEUES = {
     'default': {
         "HOST": env.str("PURLDB_REDIS_HOST", default="localhost"),
@@ -316,3 +317,8 @@
         "DEFAULT_TIMEOUT": env.int("PURLDB_REDIS_DEFAULT_TIMEOUT", default=360),
     }
 }
+
+PURLDB_ASYNC = env.bool("PURLDB_ASYNC", default=False)
+if not PURLDB_ASYNC:
+    for queue_config in RQ_QUEUES.values():
+        queue_config["ASYNC"] = False