Testing fixes

backend/entityservice/views/objectstore.py

@@ -40,32 +40,28 @@ def authorize_external_upload(project_id):
 
     log, parent_span = bind_log_and_span(project_id)
 
-    log.info("Authorizing external upload")
+    log.debug("Authorizing external upload")
     token = precheck_upload_token(project_id, headers, parent_span)
     log.debug(f"Update token is valid")
     with db.DBConn() as conn:
         dp_id = db.get_dataprovider_id(conn, token)
         log = log.bind(dpid=dp_id)
-        log.info("Got dp ID")
 
     with opentracing.tracer.start_span('assume-role-request', child_of=parent_span):
         client = connect_to_upload_object_store()
         client.set_app_info("anonlink", "development version")
 
         bucket_name = config.UPLOAD_OBJECT_STORE_BUCKET
         path = f"{project_id}/{dp_id}"
-        log.info(f"Retrieving temporary credentials for bucket/path: '{bucket_name}/{path}'")
+        log.info(f"Retrieving temporary object store credentials for path: '{bucket_name}/{path}'")
 
         credentials_provider = AssumeRoleProvider(client,
                                                   Policy=_get_upload_policy(bucket_name, path=path),
                                                   DurationSeconds=config.UPLOAD_OBJECT_STORE_STS_DURATION)
         credential_values = Credentials(provider=credentials_provider).get()
         expiry = credentials_provider._expiry._expiration
 
-        log.info("Retrieved temporary credentials:")
-        log.info(expiry)
-        log.info(credential_values.access_key)
-        log.info(credential_values.secret_key)
+        log.info("Retrieved temporary credentials")
 
     credentials_json = ObjectStoreCredentials().dump(credential_values)
     log.debug("Temp credentials", **credentials_json)

e2etests/tests/conftest.py

@@ -195,3 +195,7 @@ def groups_project(request, requests):
         [(t, 1) for t in PROJECT_RESULT_TYPES_NP]))
 def invalid_result_type_number_parties(request):
     yield request.param
+
+@pytest.fixture
+def binary_test_file_path(request):
+    return os.path.join(os.path.dirname(os.path.realpath(__file__)), 'testdata/clks_128B_1k.bin')

e2etests/tests/test_project_uploads.py

@@ -1,6 +1,7 @@
 import time
 import os
 import pytest
+from minio import Minio
 
 from e2etests.config import url
 from e2etests.util import (
@@ -23,12 +24,12 @@ def test_project_single_party_data_uploaded(requests, valid_project_params):
             'clks': generate_json_serialized_clks(100)
         }
     )
-    assert r.status_code == 201
+    assert r.status_code == 201, r.text
     upload_response = r.json()
     assert 'receipt_token' in upload_response
 
 
-def test_project_external_data_uploaded(requests, valid_project_params):
+def test_project_external_data_uploaded(requests, valid_project_params, binary_test_file_path):
     new_project_data = requests.post(url + 'projects',
                                      json={
                                          'schema': {},
@@ -38,12 +39,30 @@ def test_project_external_data_uploaded(requests, valid_project_params):
         url + 'projects/{}/authorize-external-upload'.format(new_project_data['project_id']),
         headers={'Authorization': new_project_data['update_tokens'][0]},
     )
-    assert r.status_code == 201
+    assert r.status_code == 200
     upload_response = r.json()
-    assert 'receipt_token' in upload_response
 
+    credentials = upload_response['credentials']
+    upload_info = upload_response['upload']
+
+    # Use Minio python client to upload data
+    mc = Minio(
+        upload_info['endpoint'],
+        access_key=credentials['AccessKeyId'],
+        secret_key=credentials['SecretAccessKey'],
+        session_token=credentials['SessionToken'],
+        region='us-east-1',
+        secure=False
+    )
+
+
+    etag = mc.fput_object(upload_info['bucket'], upload_info['path'] + "/test", binary_test_file_path)
 
-def test_project_binary_data_uploaded(requests, valid_project_params):
+    # Later - once the upload endpoint is complete notify the server
+    # of the uploaded data
+
+
+def test_project_binary_data_uploaded(requests, valid_project_params, binary_test_file_path):
     new_project_data = requests.post(url + '/projects',
                                      json={
                                          'schema': {},
@@ -53,12 +72,10 @@ def test_project_binary_data_uploaded(requests, valid_project_params):
     expected_number_parties = get_expected_number_parties(valid_project_params)
     assert len(update_tokens) == expected_number_parties
 
-    small_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'testdata/clks_128B_1k.bin')
-
     for token in update_tokens:
         upload_binary_data_from_file(
             requests,
-            small_file_path, new_project_data['project_id'], token, 1000)
+            binary_test_file_path, new_project_data['project_id'], token, 1000)
 
     run_id = post_run(requests, new_project_data, 0.99)
     result = get_run_result(requests, new_project_data, run_id, wait=True)

e2etests/tests/test_uploads.py

@@ -21,6 +21,7 @@ def test_get_auth_credentials(self, requests, a_project):
             credentials = raw_json['credentials']
             assert "upload" in raw_json
 
+            minio_endpoint = raw_json['upload']['endpoint']
             bucket_name = raw_json['upload']['bucket']
             allowed_path = raw_json['upload']['path']
 
@@ -29,7 +30,7 @@ def test_get_auth_credentials(self, requests, a_project):
 
             # Test we can create and use these credentials via a Minio client
             restricted_mc_client = minio.Minio(
-                "localhost:9000",
+                minio_endpoint,
                 credentials['AccessKeyId'],
                 credentials['SecretAccessKey'],
                 credentials['SessionToken'],