Initial E2E test (CrunchyData#2808)

* Add initial Kuttl e2e tests

This change adds a quick end-to-end test that use the kuttl test
framework. The layout for kuttl tests is `testing/kuttl/e2e` with
individual tests in subdirectories of the `e2e` directory.

Run test(s) by using the `make check-kuttl` target.

The initial creates a cluster then checks the postgrescluster status and
that the initial backup completes.  

the subsequent test uses the above cluster and 2 scripts to run a portforward
and to connect to psql and run select version() to return the psql version


Co-authored-by: ValClarkson <[email protected]>

Author: jmckulk

Makefile

@@ -62,6 +62,7 @@ GO ?= go
 GO_BUILD = $(GO_CMD) build -trimpath
 GO_CMD = $(GO_ENV) $(GO)
 GO_TEST ?= $(GO) test
+KUTTL_TEST ?= kuttl test
 
 # Disable optimizations if creating a debug build
 ifeq ("$(DEBUG_BUILD)", "true")
@@ -202,6 +203,11 @@ check-envtest-existing:
 	USE_EXISTING_CLUSTER=true $(GO_TEST) -count=1 -cover -p=1 -tags=envtest ./...
 	${PGO_KUBE_CLIENT} delete -k ./config/dev
 
+# Expects operator to be running
+.PHONY: check-kuttl
+check-kuttl:
+	${PGO_KUBE_CLIENT} ${KUTTL_TEST} \
+		--config testing/kuttl/kuttl-test.yaml
 
 .PHONY: check-generate
 check-generate: generate-crd generate-deepcopy generate-rbac

testing/kuttl/README.md

@@ -0,0 +1,27 @@
+# Kuttl
+
+## Installing
+Docs for install: https://kuttl.dev/docs/cli.html#setup-the-kuttl-kubectl-plugin
+
+Options:
+  - Download and install the binary
+  - Install the `kubectl krew` [plugin manager](https://github.com/kubernetes-sigs/krew)
+    and `kubectl krew install kuttl`
+## Cheat sheet
+
+### Run test suite
+
+Make sure that the operator is running in your kubernetes environment and that your `kubeconfig` is
+set up. Then run the make target:
+
+```
+make check-kuttl
+```
+
+### Running a single test
+A single test is considered to be one directory under `kuttl/e2e`, for example
+`kuttl/e2e/restore` would run the `restore` test.
+
+There are two ways to run a single test in isolation: 
+- using an env var with the make target: `KUTTL_TEST='kuttl test --test <test-name>' make check-kuttl`
+- using `kubectl kuttl --test` flag: `kubectl kuttl test testing/kuttl/e2e --test <test-name>`

testing/kuttl/e2e/cluster-start/00-assert.yaml

@@ -0,0 +1,19 @@
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: cluster-start
+status:
+  instances:
+    - name: instance1
+      readyReplicas: 1
+      replicas: 1
+      updatedReplicas: 1
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: cluster-start
+    postgres-operator.crunchydata.com/pgbackrest-backup: replica-create
+status:
+  succeeded: 1

testing/kuttl/e2e/cluster-start/00-cluster.yaml

@@ -0,0 +1,27 @@
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: cluster-start
+spec:
+  image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:centos8-13.4-1
+  postgresVersion: 13
+  instances:
+    - name: instance1
+      dataVolumeClaimSpec:
+        accessModes:
+        - "ReadWriteOnce"
+        resources:
+          requests:
+            storage: 1Gi
+  backups:
+    pgbackrest:
+      image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:centos8-2.35-0
+      repos:
+      - name: repo1
+        volume:
+          volumeClaimSpec:
+            accessModes:
+            - "ReadWriteOnce"
+            resources:
+              requests:
+                storage: 1Gi

testing/kuttl/e2e/cluster-start/01-psql-connect.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: CLUSTER=cluster-start $HOME/postgres-operator/testing/kuttl/script/port-forward.sh
+  background: true
+# this sleep is needed when running in cloud environments to ensure connection-psql.sh
+# will not fail when attempting the port-forward and has suffient time to connect, may 
+# not be needed when running locally
+- script: sleep 10
+- script: CLUSTER=cluster-start $HOME/postgres-operator/testing/kuttl/script/connect-psql.sh

testing/kuttl/kuttl-test.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+- testing/kuttl/e2e/
+timeout: 120
+parallel: 2
+# by default kuttl will run in a generated namespace to override
+# that functionality simply uncomment the line below and replace
+# postgres-operator with the desired namespace to run in.
+# namespace: postgres-operator
+suppress: 
+- events
+

testing/kuttl/script/connect-psql.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+CLUSTER=${CLUSTER:-default}
+PG_CLUSTER_USER_SECRET_NAME=${PG_CLUSTER_USER_SECRET_NAME:-$CLUSTER-pguser-$CLUSTER}
+
+PGPASSWORD=$(kubectl get secrets -n $NAMESPACE "${PG_CLUSTER_USER_SECRET_NAME}" -o go-template='{{.data.password | base64decode}}') \
+PGUSER=$(kubectl get secrets -n $NAMESPACE "${PG_CLUSTER_USER_SECRET_NAME}" -o go-template='{{.data.user | base64decode}}') \
+PGDATABASE=$(kubectl get secrets -n $NAMESPACE "${PG_CLUSTER_USER_SECRET_NAME}" -o go-template='{{.data.dbname | base64decode}}') \
+psql -h localhost -c "select version();"

testing/kuttl/script/port-forward.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+CLUSTER=${CLUSTER:-default}
+export PG_CLUSTER_PRIMARY_POD=$(kubectl get pod -n $NAMESPACE -o name -l postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master)
+kubectl -n $NAMESPACE port-forward "${PG_CLUSTER_PRIMARY_POD}" 5432:5432