Use secure connection to minio by default

azure-pipelines.yml

@@ -298,14 +298,14 @@ stages:
           namespace: $(NAMESPACE)
           releaseName: $(DEPLOYMENT)
           overrides: |
-            api.app.debug:true
             global.postgresql.postgresqlPassword:notaproductionpassword
             api.ingress.enabled:false
             workers.replicaCount:4
             api.www.image.repository:$(frontendImageName)
             api.app.image.repository:$(backendImageName)
             api.dbinit.image.repository:$(backendImageName)
             workers.image.repository:$(backendImageName)
+            anonlink.objectstore.secure:false
 
       - task: KubernetesManifest@0
         displayName: Deploy K8s manifest

backend/entityservice/api_def/openapi.yaml

@@ -726,11 +726,15 @@ paths:
 components:
   parameters:
     token:
+      in: header
+      description: |
+        Authorization token required for each endpoint. This may be an upload token or a result token,
+        both are provided by the `POST /projects` endpoint.
       required: true
       schema:
         type: string
       name: Authorization
-      in: header
+
     project_id:
       in: path
       name: "project_id"
@@ -1127,6 +1131,10 @@ components:
             endpoint:
               type: string
               description: Hostname, and port of object store. E.g. minio.anonlink.example.com:9000
+            secure:
+              type: boolean
+              description: A secure connection to the object store is required.
+              default: true
             bucket:
               type: string
               description: Target bucket

backend/entityservice/integrationtests/redistests/test_status.py

@@ -1,8 +1,5 @@
 import time
 
-import redis
-from pytest import raises
-
 from entityservice.cache import connect_to_redis, get_status, set_status
 
 
@@ -30,4 +27,5 @@ def test_set_status(self):
         time.sleep(0.2)
         updated_status = get_status()
         assert 'testkey' in updated_status
+        set_status(original_status)
 

backend/entityservice/object_store.py

@@ -30,7 +30,7 @@ def connect_to_upload_object_store():
         config.UPLOAD_OBJECT_STORE_ACCESS_KEY,
         config.UPLOAD_OBJECT_STORE_SECRET_KEY,
         region="us-east-1",
-        secure=False
+        secure=config.UPLOAD_OBJECT_STORE_SECURE
     )
     mc.set_app_info("anonlink-upload", "minio client for uploads")
     logger.debug("Connected to minio upload account")

backend/entityservice/settings.py

@@ -33,6 +33,7 @@ class Config(object):
     UPLOAD_OBJECT_STORE_ENABLED = os.getenv('UPLOAD_OBJECT_STORE_ENABLED', 'true').lower() == "true"
     UPLOAD_OBJECT_STORE_STS_DURATION = int(os.getenv('UPLOAD_OBJECT_STORE_STS_DURATION', '43200'))
     UPLOAD_OBJECT_STORE_SERVER = os.getenv('UPLOAD_OBJECT_STORE_SERVER', MINIO_SERVER)
+    UPLOAD_OBJECT_STORE_SECURE = os.getenv('UPLOAD_OBJECT_STORE_SECURE', 'true') == 'true'
     UPLOAD_OBJECT_STORE_ACCESS_KEY = os.getenv('UPLOAD_OBJECT_STORE_ACCESS_KEY', '')
     UPLOAD_OBJECT_STORE_SECRET_KEY = os.getenv('UPLOAD_OBJECT_STORE_SECRET_KEY', '')
     UPLOAD_OBJECT_STORE_BUCKET = os.getenv('UPLOAD_OBJECT_STORE_BUCKET', 'anonlink-uploads')

backend/entityservice/views/objectstore.py

@@ -77,7 +77,8 @@ def authorize_external_upload(project_id):
         "credentials": credentials_json,
         "upload": {
             "endpoint": config.UPLOAD_OBJECT_STORE_SERVER,
+            "secure": config.UPLOAD_OBJECT_STORE_SECURE,
             "bucket": bucket_name,
             "path": f"{project_id}/{dp_id}"
         }
-    }
+    }, 201

deployment/entity-service/Chart.yaml

@@ -11,3 +11,4 @@ maintainers:
     email: [email protected]
     url: https://data61.csiro.au
 icon: https://s3-us-west-2.amazonaws.com/slack-files2/avatars/2016-04-11/33560836053_df0d62a81bf32f53df00_72.png
+apiVersion: v1

deployment/entity-service/templates/configmap.yaml

@@ -32,6 +32,13 @@ data:
   MINIO_BUCKET: {{ required "minio.defaultBucket.name is required." .Values.minio.defaultBucket.name | quote }}
 
   UPLOAD_OBJECT_STORE_ENABLED: {{ .Values.anonlink.objectstore.uploadEnabled | quote }}
+
+  {{ if .Values.minio.ingress.enabled }}
+  UPLOAD_OBJECT_STORE_SERVER: {{ index .Values.minio.ingress.hosts 0 }}
+  {{ end }}
+
+  UPLOAD_OBJECT_STORE_SECURE: {{ .Values.anonlink.objectstore.secure | quote }}
+
   UPLOAD_OBJECT_STORE_BUCKET: {{ required "anonlink.objectstore.uploadBucket.name is required." .Values.anonlink.objectstore.uploadBucket.name | quote }}
 
   {{ if .Values.provision.postgresql }}

deployment/entity-service/values.yaml

@@ -9,15 +9,18 @@ anonlink:
   config: {}
 
   objectstore:
-    uploadEnabled: "true"
+    uploadEnabled: true
+
+    ## Allow only secure connection to the object store
+    secure: true
+
     # TODO can't leave defaults once minio exposed
     uploadAccessKey: "EXAMPLE_UPLOAD_KEY"
     uploadSecretKey: "EXAMPLE_UPLOAD_SECRET"
 
     uploadBucket:
       name: "uploads"
 
-
 api:
 
   ## Deployment component name
@@ -100,7 +103,7 @@ api:
 
   ## A job that creates an upload only object store user.
   objectstoreinit:
-    enabled: "true"
+    enabled: true
 
     image:
       repository: minio/mc
@@ -117,12 +120,13 @@ api:
      ## To handle large uploads we increase the proxy buffer size
      ingress.kubernetes.io/proxy-body-size: 4096m
      nginx.ingress.kubernetes.io/proxy-body-size: 4096m
-     ## Redirect to ssl
+
      nginx.ingress.kubernetes.io/proxy-connect-timeout: "60"
      nginx.ingress.kubernetes.io/proxy-send-timeout: "60"
      nginx.ingress.kubernetes.io/proxy-read-timeout: "60"
      ## Example ingress annotations for certmanager
      certmanager.k8s.io/cluster-issuer: letsencrypt-cluster-issuer
+     ## Redirect to ssl
      ingress.kubernetes.io/force-ssl-redirect: "true"
 
     ## Entity Service API Ingress hostnames

e2etests/tests/test_project_uploads.py

@@ -39,7 +39,7 @@ def test_project_external_data_uploaded(requests, valid_project_params, binary_t
         url + 'projects/{}/authorize-external-upload'.format(new_project_data['project_id']),
         headers={'Authorization': new_project_data['update_tokens'][0]},
     )
-    assert r.status_code == 200
+    assert r.status_code == 201
     upload_response = r.json()
 
     credentials = upload_response['credentials']
@@ -52,10 +52,9 @@ def test_project_external_data_uploaded(requests, valid_project_params, binary_t
         secret_key=credentials['SecretAccessKey'],
         session_token=credentials['SessionToken'],
         region='us-east-1',
-        secure=False
+        secure=upload_info['secure']
     )
 
-
     etag = mc.fput_object(upload_info['bucket'], upload_info['path'] + "/test", binary_test_file_path)
 
     # Later - once the upload endpoint is complete notify the server

e2etests/tests/test_uploads.py

@@ -22,6 +22,7 @@ def test_get_auth_credentials(self, requests, a_project):
             assert "upload" in raw_json
 
             minio_endpoint = raw_json['upload']['endpoint']
+            minio_secure = raw_json['upload']['secure']
             bucket_name = raw_json['upload']['bucket']
             allowed_path = raw_json['upload']['path']
 
@@ -35,7 +36,7 @@ def test_get_auth_credentials(self, requests, a_project):
                 credentials['SecretAccessKey'],
                 credentials['SessionToken'],
                 region='us-east-1',
-                secure=False
+                secure=minio_secure
             )
 
             # Client shouldn't be able to list buckets

tools/ci.yml

@@ -21,6 +21,7 @@ services:
       - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - UPLOAD_OBJECT_STORE_ACCESS_KEY=EXAMPLE_UPLOAD_ACCESS_KEY
       - UPLOAD_OBJECT_STORE_SECRET_KEY=EXAMPLE_UPLOAD_SECRET_ACCESS_KEY
+      - UPLOAD_OBJECT_STORE_SECURE=false
       - INITIAL_DELAY=5
     command: dockerize -wait tcp://db:5432 -wait tcp://nginx:8851/api/v1/status -timeout 5m
              /bin/sh -c "sleep 5 && python -m pytest -n 1 entityservice/integrationtests --junitxml=testResults.xml -x"

tools/docker-compose.yml

@@ -44,6 +44,7 @@ services:
       - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
       - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - UPLOAD_OBJECT_STORE_BUCKET=uploads
+      - UPLOAD_OBJECT_STORE_SECURE=false
       - UPLOAD_OBJECT_STORE_ACCESS_KEY=EXAMPLE_UPLOAD_ACCESS_KEY
       - UPLOAD_OBJECT_STORE_SECRET_KEY=EXAMPLE_UPLOAD_SECRET_ACCESS_KEY
       - FLASK_DB_MIN_CONNECTIONS=1