Save scan results to temporary files #362

    * Create new volume to store temporary files to share between Docker services

Signed-off-by: Jono Yang <jyang@nexb.com>

docker-compose_purldb.yml

@@ -28,6 +28,7 @@ services:
     volumes:
       - /etc/purldb/:/etc/purldb/
       - static:/var/purldb/static/
+      - temp_data:/tmp/minecode/
     depends_on:
       - db
 
@@ -113,6 +114,7 @@ services:
       - docker_purldb.env
     volumes:
       - /etc/purldb/:/etc/purldb/
+      - temp_data:/tmp/minecode/
     depends_on:
       - redis
       - db
@@ -141,3 +143,4 @@ volumes:
   db_data:
   static:
   redis_data:
+  temp_data:

minecode/api.py

@@ -25,6 +25,7 @@
 from minecode.models import PriorityResourceURI, ResourceURI, ScannableURI
 from minecode.permissions import IsScanQueueWorkerAPIUser
 from minecode.utils import validate_uuid
+from minecode.utils import get_temp_file
 
 
 class ResourceURISerializer(serializers.ModelSerializer):
@@ -202,9 +203,24 @@ def update_status(self, request, *args, **kwargs):
             scan_results_file = request.data.get('scan_results_file')
             scan_summary_file = request.data.get('scan_summary_file')
             project_extra_data = request.data.get('project_extra_data')
-            job = scannable_uri.process_scan_results(
-                scan_results_file=scan_results_file,
-                scan_summary_file=scan_summary_file,
+
+            # Save results to temporary files
+            scan_results_location = get_temp_file(
+                file_name='scan_results',
+                extension='.json'
+            )
+            scan_summary_location = get_temp_file(
+                file_name='scan_summary',
+                extension='.json'
+            )
+            with open(scan_results_location, 'wb') as f:
+                f.write(scan_results_file.read())
+            with open(scan_summary_location, 'wb') as f:
+                f.write(scan_summary_file.read())
+
+            _ = scannable_uri.process_scan_results(
+                scan_results_location=scan_results_location,
+                scan_summary_location=scan_summary_location,
                 project_extra_data=project_extra_data
             )
             msg = {

minecode/models.py

@@ -760,16 +760,21 @@ def save(self, *args, **kwargs):
         self.normalize_fields()
         super(ScannableURI, self).save(*args, **kwargs)
 
-    def process_scan_results(self, scan_results_file, scan_summary_file, project_extra_data):
+    def process_scan_results(
+        self,
+        scan_results_location,
+        scan_summary_location,
+        project_extra_data
+    ):
         from minecode import tasks
 
         self.scan_status = self.SCAN_COMPLETED
         self.save()
         job = django_rq.enqueue(
             tasks.process_scan_results,
             scannable_uri=self,
-            scan_results_file=scan_results_file,
-            scan_summary_file=scan_summary_file,
+            scan_results_location=scan_results_location,
+            scan_summary_location=scan_summary_location,
             project_extra_data=project_extra_data,
         )
         return job

minecode/tasks.py

@@ -9,23 +9,27 @@
 
 import json
 
+from commoncode.fileutils import delete
+
 from minecode.management.indexing import index_package
 from minecode.models import ScannableURI
 
 
 def process_scan_results(
     scannable_uri,
-    scan_results_file,
-    scan_summary_file,
+    scan_results_location,
+    scan_summary_location,
     project_extra_data,
 ):
     """
     Indexes the scan results from `scan_results_file`, `scan_summary_file`, and
     `project_extra_data` for the Package related to `scannable_uri`
     """
 
-    scan_data = json.load(scan_results_file)
-    summary_data = json.load(scan_summary_file)
+    with open(scan_results_location) as f:
+        scan_data = json.load(f)
+    with open(scan_summary_location) as f:
+        summary_data = json.load(f)
     project_extra_data = json.loads(project_extra_data)
 
     indexing_errors = index_package(
@@ -44,3 +48,7 @@ def process_scan_results(
         scannable_uri.scan_status = ScannableURI.SCAN_INDEXED
     scannable_uri.wip_date = None
     scannable_uri.save()
+
+    # Clean up after indexing has ended
+    delete(scan_results_location)
+    delete(scan_summary_location)