Expose minio

azure-pipelines.yml

@@ -298,14 +298,22 @@ stages:
           namespace: $(NAMESPACE)
           releaseName: $(DEPLOYMENT)
           overrides: |
-            api.app.debug:true
             global.postgresql.postgresqlPassword:notaproductionpassword
             api.ingress.enabled:false
             workers.replicaCount:4
             api.www.image.repository:$(frontendImageName)
+            api.www.image.pullPolicy:Always
             api.app.image.repository:$(backendImageName)
+            api.app.image.pullPolicy:Always
             api.dbinit.image.repository:$(backendImageName)
             workers.image.repository:$(backendImageName)
+            workers.image.pullPolicy:Always
+            anonlink.objectstore.secure:false
+            anonlink.objectstore.uploadEnabled:true
+            anonlink.objectstore.uploadAccessKey:testUploadAccessKey
+            anonlink.objectstore.uploadSecretKey:testUploadSecretKey
+            minio.accessKey:testMinioAccessKey
+            minio.secretKey:testMinioSecretKey
 
       - task: KubernetesManifest@0
         displayName: Deploy K8s manifest

backend/entityservice/api_def/openapi.yaml

@@ -6,7 +6,7 @@
 # Some style notes:
 # - This file is used by ReDoc, which allows GitHub Flavored Markdown in
 #   descriptions.
-openapi: 3.0.0
+openapi: 3.0.2
 info:
   version: '1.13'
   title: Entity Matching API
@@ -726,11 +726,15 @@ paths:
 components:
   parameters:
     token:
+      in: header
+      description: |
+        Authorization token required for each endpoint. This may be an upload token or a result token,
+        both are provided by the `POST /projects` endpoint.
       required: true
       schema:
         type: string
       name: Authorization
-      in: header
+
     project_id:
       in: path
       name: "project_id"
@@ -1127,6 +1131,10 @@ components:
             endpoint:
               type: string
               description: Hostname, and port of object store. E.g. minio.anonlink.example.com:9000
+            secure:
+              type: boolean
+              description: A secure connection to the object store is required.
+              default: true
             bucket:
               type: string
               description: Target bucket

backend/entityservice/integrationtests/redistests/test_status.py

@@ -1,8 +1,5 @@
 import time
 
-import redis
-from pytest import raises
-
 from entityservice.cache import connect_to_redis, get_status, set_status
 
 
@@ -30,4 +27,5 @@ def test_set_status(self):
         time.sleep(0.2)
         updated_status = get_status()
         assert 'testkey' in updated_status
+        set_status(original_status)
 

backend/entityservice/object_store.py

@@ -30,7 +30,7 @@ def connect_to_upload_object_store():
         config.UPLOAD_OBJECT_STORE_ACCESS_KEY,
         config.UPLOAD_OBJECT_STORE_SECRET_KEY,
         region="us-east-1",
-        secure=False
+        secure=config.UPLOAD_OBJECT_STORE_SECURE
     )
     mc.set_app_info("anonlink-upload", "minio client for uploads")
     logger.debug("Connected to minio upload account")

backend/entityservice/settings.py

@@ -33,6 +33,7 @@ class Config(object):
     UPLOAD_OBJECT_STORE_ENABLED = os.getenv('UPLOAD_OBJECT_STORE_ENABLED', 'true').lower() == "true"
     UPLOAD_OBJECT_STORE_STS_DURATION = int(os.getenv('UPLOAD_OBJECT_STORE_STS_DURATION', '43200'))
     UPLOAD_OBJECT_STORE_SERVER = os.getenv('UPLOAD_OBJECT_STORE_SERVER', MINIO_SERVER)
+    UPLOAD_OBJECT_STORE_SECURE = os.getenv('UPLOAD_OBJECT_STORE_SECURE', 'true') == 'true'
     UPLOAD_OBJECT_STORE_ACCESS_KEY = os.getenv('UPLOAD_OBJECT_STORE_ACCESS_KEY', '')
     UPLOAD_OBJECT_STORE_SECRET_KEY = os.getenv('UPLOAD_OBJECT_STORE_SECRET_KEY', '')
     UPLOAD_OBJECT_STORE_BUCKET = os.getenv('UPLOAD_OBJECT_STORE_BUCKET', 'anonlink-uploads')

backend/entityservice/views/objectstore.py

@@ -77,7 +77,8 @@ def authorize_external_upload(project_id):
         "credentials": credentials_json,
         "upload": {
             "endpoint": config.UPLOAD_OBJECT_STORE_SERVER,
+            "secure": config.UPLOAD_OBJECT_STORE_SECURE,
             "bucket": bucket_name,
             "path": f"{project_id}/{dp_id}"
         }
-    }
+    }, 201

deployment/entity-service/Chart.yaml

@@ -11,3 +11,4 @@ maintainers:
     email: [email protected]
     url: https://data61.csiro.au
 icon: https://s3-us-west-2.amazonaws.com/slack-files2/avatars/2016-04-11/33560836053_df0d62a81bf32f53df00_72.png
+apiVersion: v1

deployment/entity-service/requirements.yaml

@@ -8,6 +8,6 @@ dependencies:
     repository: https://kubernetes-charts.storage.googleapis.com
     condition: provision.minio
   - name: postgresql
-    version: 8.3.0
-    repository: https://kubernetes-charts.storage.googleapis.com
+    version: 8.9.1
+    repository: https://charts.bitnami.com/bitnami
     condition: provision.postgresql

deployment/entity-service/templates/NOTES.txt

@@ -1,37 +1,48 @@
 
-*- Confidential Computing Anonlink Service Deployed -*
+*- Anonlink Service Deployed -*
 
-Soon you should be able to visit the entity service api.
+The Anonlink Entity Service can be accessed via port {{ .Values.api.service.servicePort }} on the
+following DNS name from within your cluster:
+{{ template "api.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
 
 1. Get the entity service URL by running:
 
-{{- if contains "NodePort" .Values.api.service.type }}
+{{- if eq .Values.api.service.type "NodePort" "ClusterIP" }}
 
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "es.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT/
+{{- if .Values.api.ingress.enabled }}
+As you're using an ingress controller you will find the Anonlink Entity Service running at
+
+    http://{{ index .Values.api.ingress.hosts 0 }}
+
+in just a moment. Note depending on your cluster set up you may have to manually configure the
+DNS entry.
+{{- end }}
+
+To access Anonlink Entity Service API from localhost:
+
+  export SVC_NAME=$(kubectl get services --namespace {{ .Release.Namespace }} -l "release={{ .Release.Name }},tier=frontend" -o jsonpath="{.items[0].metadata.name}")
+  kubectl port-forward svc/$SVC_NAME 8080:80
+
+Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/
+
+And visit http://127.0.0.1:8080
 
 {{- else if contains "LoadBalancer" .Values.api.service.type }}
 
-  It may take a few minutes for the LoadBalancer IP to be available.
+  It may take a few minutes for the LoadBalancer's public IP to be available.
   Watch the status with:
 
       kubectl get svc -w entityservice-api --namespace {{ .Release.Namespace }}
 
       export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} entityservice-api -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
       echo http://$SERVICE_IP/
 
-{{- else if contains "ClusterIP"  .Values.api.service.type }}
-
-If you're using an ingress controller (the default) you may have to manually
-add the DNS entry for http://{{ .Values.api.ingress.hosts }} now.
-
-Alternatively you can port forward the entity service API to your
-local machine:
+{{- end }}
 
-  export SVC_NAME=$(kubectl get services --namespace {{ .Release.Namespace }} -l "release={{ .Release.Name }},tier=frontend" -o jsonpath="{.items[0].metadata.name}")
-  kubectl port-forward SVC_NAME 8080:80
+You may wish to connect to the API using anonlink-client. https://github.com/data61/anonlink-client/
+The client can be installed using pip:
 
-And visit http://127.0.0.1:8080/
+    $ pip install anonlinkclient
+    $ anonlink status --server=<SERVER>
 
-{{- end }}
+Issues with the Anonlink Entity Service can be created on Github - https://github.com/data61/anonlink-entity-service/

deployment/entity-service/templates/configmap.yaml

@@ -32,6 +32,13 @@ data:
   MINIO_BUCKET: {{ required "minio.defaultBucket.name is required." .Values.minio.defaultBucket.name | quote }}
 
   UPLOAD_OBJECT_STORE_ENABLED: {{ .Values.anonlink.objectstore.uploadEnabled | quote }}
+
+  {{ if .Values.minio.ingress.enabled }}
+  UPLOAD_OBJECT_STORE_SERVER: {{ index .Values.minio.ingress.hosts 0 }}
+  {{ end }}
+
+  UPLOAD_OBJECT_STORE_SECURE: {{ .Values.anonlink.objectstore.secure | quote }}
+
   UPLOAD_OBJECT_STORE_BUCKET: {{ required "anonlink.objectstore.uploadBucket.name is required." .Values.anonlink.objectstore.uploadBucket.name | quote }}
 
   {{ if .Values.provision.postgresql }}

deployment/entity-service/templates/init-db-job.yaml

@@ -33,8 +33,6 @@ spec:
               secretKeyRef:
                 name: {{ template "es.fullname" . }}
                 key: postgresPassword
-          - name: DEBUG
-            value: {{default "false" .Values.api.app.debug | quote }}
           - name: FLASK_APP
             value: entityservice
         command:

deployment/entity-service/values.yaml

@@ -9,14 +9,24 @@ anonlink:
   config: {}
 
   objectstore:
-    uploadEnabled: "true"
-    # TODO can't leave defaults once minio exposed
+    ## Toggle the feature providing client's with restricted upload access to the object store.
+    ## By default we don't expose the Minio object store, which is required for clients to upload
+    ## via the object store. See section `minio.ingress` to create an ingress for minio.
+    uploadEnabled: true
+
+    ## Object store credentials used to grant temporary upload access to clients
+    ## Will be created with an "upload only" policy for a upload bucket if using the default
+    ## MINIO provisioning.
     uploadAccessKey: "EXAMPLE_UPLOAD_KEY"
     uploadSecretKey: "EXAMPLE_UPLOAD_SECRET"
 
+    ## The bucket for client uploads.
     uploadBucket:
       name: "uploads"
 
+    ## Allow only secure connections to the object store.
+    secure: true
+
 
 api:
 
@@ -65,8 +75,7 @@ api:
     image:
       repository: data61/anonlink-app
       tag: "v1.13.0-beta2"
-      # pullPolicy: IfNotPresent
-      pullPolicy: Always
+      pullPolicy: IfNotPresent
 
 
     ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/
@@ -78,8 +87,6 @@ api:
         cpu: 500m
         memory: 512Mi
 
-    debug: "false"
-
   dbinit:
     enabled: "true"
 
@@ -100,7 +107,7 @@ api:
 
   ## A job that creates an upload only object store user.
   objectstoreinit:
-    enabled: "true"
+    enabled: true
 
     image:
       repository: minio/mc
@@ -117,12 +124,13 @@ api:
      ## To handle large uploads we increase the proxy buffer size
      ingress.kubernetes.io/proxy-body-size: 4096m
      nginx.ingress.kubernetes.io/proxy-body-size: 4096m
-     ## Redirect to ssl
+
      nginx.ingress.kubernetes.io/proxy-connect-timeout: "60"
      nginx.ingress.kubernetes.io/proxy-send-timeout: "60"
      nginx.ingress.kubernetes.io/proxy-read-timeout: "60"
      ## Example ingress annotations for certmanager
      certmanager.k8s.io/cluster-issuer: letsencrypt-cluster-issuer
+     ## Redirect to ssl
      ingress.kubernetes.io/force-ssl-redirect: "true"
 
     ## Entity Service API Ingress hostnames
@@ -346,10 +354,11 @@ minio:
   ## Configure the object storage
   ## https://github.com/helm/charts/blob/master/stable/minio/values.yaml
 
-  ## Default access credentials for the object store
-  # TODO remove defaults once minio exposed to interwebs
-  accessKey: "exampleMinioAccessKey"
-  secretKey: "exampleMinioSecretKet"
+  ## Root access credentials for the object store
+  ## Note no defaults are provided to help prevent data breaches where
+  ## the object store is exposed to the internet
+  #accessKey: "exampleMinioAccessKey"
+  #secretKey: "exampleMinioSecretKet"
 
   ## Settings required for connecting to another object store, the server is ignored
   ## if provisioning minio during deployment.

e2etests/tests/test_project_uploads.py

@@ -39,7 +39,7 @@ def test_project_external_data_uploaded(requests, valid_project_params, binary_t
         url + 'projects/{}/authorize-external-upload'.format(new_project_data['project_id']),
         headers={'Authorization': new_project_data['update_tokens'][0]},
     )
-    assert r.status_code == 200
+    assert r.status_code == 201
     upload_response = r.json()
 
     credentials = upload_response['credentials']
@@ -52,10 +52,9 @@ def test_project_external_data_uploaded(requests, valid_project_params, binary_t
         secret_key=credentials['SecretAccessKey'],
         session_token=credentials['SessionToken'],
         region='us-east-1',
-        secure=False
+        secure=upload_info['secure']
     )
 
-
     etag = mc.fput_object(upload_info['bucket'], upload_info['path'] + "/test", binary_test_file_path)
 
     # Later - once the upload endpoint is complete notify the server

e2etests/tests/test_uploads.py

@@ -15,13 +15,14 @@ def test_get_auth_credentials(self, requests, a_project):
             res = requests.get(url + f"projects/{pid}/authorize-external-upload",
                                headers={'Authorization': a_project['update_tokens'][dp_index]})
 
-            assert res.status_code == 200
+            assert res.status_code == 201
             raw_json = res.json()
             assert "credentials" in raw_json
             credentials = raw_json['credentials']
             assert "upload" in raw_json
 
             minio_endpoint = raw_json['upload']['endpoint']
+            minio_secure = raw_json['upload']['secure']
             bucket_name = raw_json['upload']['bucket']
             allowed_path = raw_json['upload']['path']
 
@@ -35,7 +36,7 @@ def test_get_auth_credentials(self, requests, a_project):
                 credentials['SecretAccessKey'],
                 credentials['SessionToken'],
                 region='us-east-1',
-                secure=False
+                secure=minio_secure
             )
 
             # Client shouldn't be able to list buckets

tools/ci.yml

@@ -21,6 +21,7 @@ services:
       - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - UPLOAD_OBJECT_STORE_ACCESS_KEY=EXAMPLE_UPLOAD_ACCESS_KEY
       - UPLOAD_OBJECT_STORE_SECRET_KEY=EXAMPLE_UPLOAD_SECRET_ACCESS_KEY
+      - UPLOAD_OBJECT_STORE_SECURE=false
       - INITIAL_DELAY=5
     command: dockerize -wait tcp://db:5432 -wait tcp://nginx:8851/api/v1/status -timeout 5m
              /bin/sh -c "sleep 5 && python -m pytest -n 1 entityservice/integrationtests --junitxml=testResults.xml -x"

tools/docker-compose.yml

@@ -44,6 +44,7 @@ services:
       - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
       - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       - UPLOAD_OBJECT_STORE_BUCKET=uploads
+      - UPLOAD_OBJECT_STORE_SECURE=false
       - UPLOAD_OBJECT_STORE_ACCESS_KEY=EXAMPLE_UPLOAD_ACCESS_KEY
       - UPLOAD_OBJECT_STORE_SECRET_KEY=EXAMPLE_UPLOAD_SECRET_ACCESS_KEY
       - FLASK_DB_MIN_CONNECTIONS=1