diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 563e79cc..9de6d7de 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -298,14 +298,22 @@ stages: namespace: $(NAMESPACE) releaseName: $(DEPLOYMENT) overrides: | - api.app.debug:true global.postgresql.postgresqlPassword:notaproductionpassword api.ingress.enabled:false workers.replicaCount:4 api.www.image.repository:$(frontendImageName) + api.www.image.pullPolicy:Always api.app.image.repository:$(backendImageName) + api.app.image.pullPolicy:Always api.dbinit.image.repository:$(backendImageName) workers.image.repository:$(backendImageName) + workers.image.pullPolicy:Always + anonlink.objectstore.secure:false + anonlink.objectstore.uploadEnabled:true + anonlink.objectstore.uploadAccessKey:testUploadAccessKey + anonlink.objectstore.uploadSecretKey:testUploadSecretKey + minio.accessKey:testMinioAccessKey + minio.secretKey:testMinioSecretKey - task: KubernetesManifest@0 displayName: Deploy K8s manifest diff --git a/backend/entityservice/api_def/openapi.yaml b/backend/entityservice/api_def/openapi.yaml index 13d5073a..11757d0f 100644 --- a/backend/entityservice/api_def/openapi.yaml +++ b/backend/entityservice/api_def/openapi.yaml @@ -6,7 +6,7 @@ # Some style notes: # - This file is used by ReDoc, which allows GitHub Flavored Markdown in # descriptions. -openapi: 3.0.0 +openapi: 3.0.2 info: version: '1.13' title: Entity Matching API @@ -726,11 +726,15 @@ paths: components: parameters: token: + in: header + description: | + Authorization token required for each endpoint. This may be an upload token or a result token, + both are provided by the `POST /projects` endpoint. required: true schema: type: string name: Authorization - in: header + project_id: in: path name: "project_id" @@ -1127,6 +1131,10 @@ components: endpoint: type: string description: Hostname, and port of object store. E.g. minio.anonlink.example.com:9000 + secure: + type: boolean + description: A secure connection to the object store is required. + default: true bucket: type: string description: Target bucket diff --git a/backend/entityservice/integrationtests/redistests/test_status.py b/backend/entityservice/integrationtests/redistests/test_status.py index ba01958e..16ac500b 100644 --- a/backend/entityservice/integrationtests/redistests/test_status.py +++ b/backend/entityservice/integrationtests/redistests/test_status.py @@ -1,8 +1,5 @@ import time -import redis -from pytest import raises - from entityservice.cache import connect_to_redis, get_status, set_status @@ -30,4 +27,5 @@ def test_set_status(self): time.sleep(0.2) updated_status = get_status() assert 'testkey' in updated_status + set_status(original_status) diff --git a/backend/entityservice/object_store.py b/backend/entityservice/object_store.py index bf754e05..83eb0f61 100644 --- a/backend/entityservice/object_store.py +++ b/backend/entityservice/object_store.py @@ -30,7 +30,7 @@ def connect_to_upload_object_store(): config.UPLOAD_OBJECT_STORE_ACCESS_KEY, config.UPLOAD_OBJECT_STORE_SECRET_KEY, region="us-east-1", - secure=False + secure=config.UPLOAD_OBJECT_STORE_SECURE ) mc.set_app_info("anonlink-upload", "minio client for uploads") logger.debug("Connected to minio upload account") diff --git a/backend/entityservice/settings.py b/backend/entityservice/settings.py index 6e0f412e..2b749cad 100644 --- a/backend/entityservice/settings.py +++ b/backend/entityservice/settings.py @@ -33,6 +33,7 @@ class Config(object): UPLOAD_OBJECT_STORE_ENABLED = os.getenv('UPLOAD_OBJECT_STORE_ENABLED', 'true').lower() == "true" UPLOAD_OBJECT_STORE_STS_DURATION = int(os.getenv('UPLOAD_OBJECT_STORE_STS_DURATION', '43200')) UPLOAD_OBJECT_STORE_SERVER = os.getenv('UPLOAD_OBJECT_STORE_SERVER', MINIO_SERVER) + UPLOAD_OBJECT_STORE_SECURE = os.getenv('UPLOAD_OBJECT_STORE_SECURE', 'true') == 'true' UPLOAD_OBJECT_STORE_ACCESS_KEY = os.getenv('UPLOAD_OBJECT_STORE_ACCESS_KEY', '') UPLOAD_OBJECT_STORE_SECRET_KEY = os.getenv('UPLOAD_OBJECT_STORE_SECRET_KEY', '') UPLOAD_OBJECT_STORE_BUCKET = os.getenv('UPLOAD_OBJECT_STORE_BUCKET', 'anonlink-uploads') diff --git a/backend/entityservice/views/objectstore.py b/backend/entityservice/views/objectstore.py index c9055046..2e3a47b0 100644 --- a/backend/entityservice/views/objectstore.py +++ b/backend/entityservice/views/objectstore.py @@ -77,7 +77,8 @@ def authorize_external_upload(project_id): "credentials": credentials_json, "upload": { "endpoint": config.UPLOAD_OBJECT_STORE_SERVER, + "secure": config.UPLOAD_OBJECT_STORE_SECURE, "bucket": bucket_name, "path": f"{project_id}/{dp_id}" } - } + }, 201 diff --git a/deployment/entity-service/Chart.yaml b/deployment/entity-service/Chart.yaml index 6871d04d..bfb82449 100644 --- a/deployment/entity-service/Chart.yaml +++ b/deployment/entity-service/Chart.yaml @@ -11,3 +11,4 @@ maintainers: email: brian.thorne@data61.csiro.au url: https://data61.csiro.au icon: https://s3-us-west-2.amazonaws.com/slack-files2/avatars/2016-04-11/33560836053_df0d62a81bf32f53df00_72.png +apiVersion: v1 diff --git a/deployment/entity-service/requirements.yaml b/deployment/entity-service/requirements.yaml index 5293b24c..89564ece 100644 --- a/deployment/entity-service/requirements.yaml +++ b/deployment/entity-service/requirements.yaml @@ -8,6 +8,6 @@ dependencies: repository: https://kubernetes-charts.storage.googleapis.com condition: provision.minio - name: postgresql - version: 8.3.0 - repository: https://kubernetes-charts.storage.googleapis.com + version: 8.9.1 + repository: https://charts.bitnami.com/bitnami condition: provision.postgresql diff --git a/deployment/entity-service/templates/NOTES.txt b/deployment/entity-service/templates/NOTES.txt index a11afec8..b1eef862 100644 --- a/deployment/entity-service/templates/NOTES.txt +++ b/deployment/entity-service/templates/NOTES.txt @@ -1,19 +1,35 @@ -*- Confidential Computing Anonlink Service Deployed -* +*- Anonlink Service Deployed -* -Soon you should be able to visit the entity service api. +The Anonlink Entity Service can be accessed via port {{ .Values.api.service.servicePort }} on the +following DNS name from within your cluster: +{{ template "api.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local 1. Get the entity service URL by running: -{{- if contains "NodePort" .Values.api.service.type }} +{{- if eq .Values.api.service.type "NodePort" "ClusterIP" }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "es.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT/ +{{- if .Values.api.ingress.enabled }} +As you're using an ingress controller you will find the Anonlink Entity Service running at + + http://{{ index .Values.api.ingress.hosts 0 }} + +in just a moment. Note depending on your cluster set up you may have to manually configure the +DNS entry. +{{- end }} + +To access Anonlink Entity Service API from localhost: + + export SVC_NAME=$(kubectl get services --namespace {{ .Release.Namespace }} -l "release={{ .Release.Name }},tier=frontend" -o jsonpath="{.items[0].metadata.name}") + kubectl port-forward svc/$SVC_NAME 8080:80 + +Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ + +And visit http://127.0.0.1:8080 {{- else if contains "LoadBalancer" .Values.api.service.type }} - It may take a few minutes for the LoadBalancer IP to be available. + It may take a few minutes for the LoadBalancer's public IP to be available. Watch the status with: kubectl get svc -w entityservice-api --namespace {{ .Release.Namespace }} @@ -21,17 +37,12 @@ Soon you should be able to visit the entity service api. export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} entityservice-api -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') echo http://$SERVICE_IP/ -{{- else if contains "ClusterIP" .Values.api.service.type }} - -If you're using an ingress controller (the default) you may have to manually -add the DNS entry for http://{{ .Values.api.ingress.hosts }} now. - -Alternatively you can port forward the entity service API to your -local machine: +{{- end }} - export SVC_NAME=$(kubectl get services --namespace {{ .Release.Namespace }} -l "release={{ .Release.Name }},tier=frontend" -o jsonpath="{.items[0].metadata.name}") - kubectl port-forward SVC_NAME 8080:80 +You may wish to connect to the API using anonlink-client. https://github.com/data61/anonlink-client/ +The client can be installed using pip: -And visit http://127.0.0.1:8080/ + $ pip install anonlinkclient + $ anonlink status --server= -{{- end }} +Issues with the Anonlink Entity Service can be created on Github - https://github.com/data61/anonlink-entity-service/ diff --git a/deployment/entity-service/templates/configmap.yaml b/deployment/entity-service/templates/configmap.yaml index 8b1723f7..434f82fd 100644 --- a/deployment/entity-service/templates/configmap.yaml +++ b/deployment/entity-service/templates/configmap.yaml @@ -32,6 +32,13 @@ data: MINIO_BUCKET: {{ required "minio.defaultBucket.name is required." .Values.minio.defaultBucket.name | quote }} UPLOAD_OBJECT_STORE_ENABLED: {{ .Values.anonlink.objectstore.uploadEnabled | quote }} + + {{ if .Values.minio.ingress.enabled }} + UPLOAD_OBJECT_STORE_SERVER: {{ index .Values.minio.ingress.hosts 0 }} + {{ end }} + + UPLOAD_OBJECT_STORE_SECURE: {{ .Values.anonlink.objectstore.secure | quote }} + UPLOAD_OBJECT_STORE_BUCKET: {{ required "anonlink.objectstore.uploadBucket.name is required." .Values.anonlink.objectstore.uploadBucket.name | quote }} {{ if .Values.provision.postgresql }} diff --git a/deployment/entity-service/templates/init-db-job.yaml b/deployment/entity-service/templates/init-db-job.yaml index 8f2ecfad..a90abf64 100644 --- a/deployment/entity-service/templates/init-db-job.yaml +++ b/deployment/entity-service/templates/init-db-job.yaml @@ -33,8 +33,6 @@ spec: secretKeyRef: name: {{ template "es.fullname" . }} key: postgresPassword - - name: DEBUG - value: {{default "false" .Values.api.app.debug | quote }} - name: FLASK_APP value: entityservice command: diff --git a/deployment/entity-service/values.yaml b/deployment/entity-service/values.yaml index ac2f3848..e09ff675 100644 --- a/deployment/entity-service/values.yaml +++ b/deployment/entity-service/values.yaml @@ -9,14 +9,24 @@ anonlink: config: {} objectstore: - uploadEnabled: "true" - # TODO can't leave defaults once minio exposed + ## Toggle the feature providing client's with restricted upload access to the object store. + ## By default we don't expose the Minio object store, which is required for clients to upload + ## via the object store. See section `minio.ingress` to create an ingress for minio. + uploadEnabled: true + + ## Object store credentials used to grant temporary upload access to clients + ## Will be created with an "upload only" policy for a upload bucket if using the default + ## MINIO provisioning. uploadAccessKey: "EXAMPLE_UPLOAD_KEY" uploadSecretKey: "EXAMPLE_UPLOAD_SECRET" + ## The bucket for client uploads. uploadBucket: name: "uploads" + ## Allow only secure connections to the object store. + secure: true + api: @@ -65,8 +75,7 @@ api: image: repository: data61/anonlink-app tag: "v1.13.0-beta2" - # pullPolicy: IfNotPresent - pullPolicy: Always + pullPolicy: IfNotPresent ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ @@ -78,8 +87,6 @@ api: cpu: 500m memory: 512Mi - debug: "false" - dbinit: enabled: "true" @@ -100,7 +107,7 @@ api: ## A job that creates an upload only object store user. objectstoreinit: - enabled: "true" + enabled: true image: repository: minio/mc @@ -117,12 +124,13 @@ api: ## To handle large uploads we increase the proxy buffer size ingress.kubernetes.io/proxy-body-size: 4096m nginx.ingress.kubernetes.io/proxy-body-size: 4096m - ## Redirect to ssl + nginx.ingress.kubernetes.io/proxy-connect-timeout: "60" nginx.ingress.kubernetes.io/proxy-send-timeout: "60" nginx.ingress.kubernetes.io/proxy-read-timeout: "60" ## Example ingress annotations for certmanager certmanager.k8s.io/cluster-issuer: letsencrypt-cluster-issuer + ## Redirect to ssl ingress.kubernetes.io/force-ssl-redirect: "true" ## Entity Service API Ingress hostnames @@ -346,10 +354,11 @@ minio: ## Configure the object storage ## https://github.com/helm/charts/blob/master/stable/minio/values.yaml - ## Default access credentials for the object store - # TODO remove defaults once minio exposed to interwebs - accessKey: "exampleMinioAccessKey" - secretKey: "exampleMinioSecretKet" + ## Root access credentials for the object store + ## Note no defaults are provided to help prevent data breaches where + ## the object store is exposed to the internet + #accessKey: "exampleMinioAccessKey" + #secretKey: "exampleMinioSecretKet" ## Settings required for connecting to another object store, the server is ignored ## if provisioning minio during deployment. diff --git a/e2etests/tests/test_project_uploads.py b/e2etests/tests/test_project_uploads.py index 4d2b83a9..3dd13ded 100644 --- a/e2etests/tests/test_project_uploads.py +++ b/e2etests/tests/test_project_uploads.py @@ -39,7 +39,7 @@ def test_project_external_data_uploaded(requests, valid_project_params, binary_t url + 'projects/{}/authorize-external-upload'.format(new_project_data['project_id']), headers={'Authorization': new_project_data['update_tokens'][0]}, ) - assert r.status_code == 200 + assert r.status_code == 201 upload_response = r.json() credentials = upload_response['credentials'] @@ -52,10 +52,9 @@ def test_project_external_data_uploaded(requests, valid_project_params, binary_t secret_key=credentials['SecretAccessKey'], session_token=credentials['SessionToken'], region='us-east-1', - secure=False + secure=upload_info['secure'] ) - etag = mc.fput_object(upload_info['bucket'], upload_info['path'] + "/test", binary_test_file_path) # Later - once the upload endpoint is complete notify the server diff --git a/e2etests/tests/test_uploads.py b/e2etests/tests/test_uploads.py index 57a1221e..7c4131bc 100644 --- a/e2etests/tests/test_uploads.py +++ b/e2etests/tests/test_uploads.py @@ -15,13 +15,14 @@ def test_get_auth_credentials(self, requests, a_project): res = requests.get(url + f"projects/{pid}/authorize-external-upload", headers={'Authorization': a_project['update_tokens'][dp_index]}) - assert res.status_code == 200 + assert res.status_code == 201 raw_json = res.json() assert "credentials" in raw_json credentials = raw_json['credentials'] assert "upload" in raw_json minio_endpoint = raw_json['upload']['endpoint'] + minio_secure = raw_json['upload']['secure'] bucket_name = raw_json['upload']['bucket'] allowed_path = raw_json['upload']['path'] @@ -35,7 +36,7 @@ def test_get_auth_credentials(self, requests, a_project): credentials['SecretAccessKey'], credentials['SessionToken'], region='us-east-1', - secure=False + secure=minio_secure ) # Client shouldn't be able to list buckets diff --git a/tools/ci.yml b/tools/ci.yml index 6e397d3a..cb3b8544 100644 --- a/tools/ci.yml +++ b/tools/ci.yml @@ -21,6 +21,7 @@ services: - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY - UPLOAD_OBJECT_STORE_ACCESS_KEY=EXAMPLE_UPLOAD_ACCESS_KEY - UPLOAD_OBJECT_STORE_SECRET_KEY=EXAMPLE_UPLOAD_SECRET_ACCESS_KEY + - UPLOAD_OBJECT_STORE_SECURE=false - INITIAL_DELAY=5 command: dockerize -wait tcp://db:5432 -wait tcp://nginx:8851/api/v1/status -timeout 5m /bin/sh -c "sleep 5 && python -m pytest -n 1 entityservice/integrationtests --junitxml=testResults.xml -x" diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index bb136f36..9e7b9cc2 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -44,6 +44,7 @@ services: - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY - UPLOAD_OBJECT_STORE_BUCKET=uploads + - UPLOAD_OBJECT_STORE_SECURE=false - UPLOAD_OBJECT_STORE_ACCESS_KEY=EXAMPLE_UPLOAD_ACCESS_KEY - UPLOAD_OBJECT_STORE_SECRET_KEY=EXAMPLE_UPLOAD_SECRET_ACCESS_KEY - FLASK_DB_MIN_CONNECTIONS=1