Merge pull request #53 from luthermonson/e2e-testing

Adding e2e Testing

Author: dramich

Dockerfile.dapper

@@ -17,6 +17,18 @@ RUN if [ "${ARCH}" == "amd64" ]; then \
         curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.15.0; \
     fi
 
+ENV K3S_VERSION v0.6.1
+#integration tests only support amd64
+RUN if [ "${ARCH}" == "amd64" ]; then \
+        curl -sL https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s > /usr/bin/k3s \
+        && chmod +x /usr/bin/k3s; \
+    fi
+
+ENV DAPPER_RUN_ARGS --privileged
+VOLUME /var/lib/rancher/k3s
+VOLUME /var/lib/cni
+VOLUME /var/log
+
 ENV DAPPER_ENV REPO TAG DRONE_TAG
 ENV DAPPER_SOURCE /go/src/github.com/rancher/terraform-controller/
 ENV DAPPER_OUTPUT ./bin ./dist

e2e/README.md

@@ -0,0 +1,59 @@
+# End to End Testing
+
+This folder contains all End to End (e2e) testing code 
+
+## Prerequisites
+
+- The e2e tests require you to run `make build-controller` at least once because the binary `./bin/terraform-controller` needs to be available to the e2e tests.
+- These tests are expecting a new clean k8s cluster and does not clean up after itself so it is expected that the cluster will be thrown away after e2e tests are complete. You will want to look at k3s/k3d to make this process smoother. Using the `make` scripts are the best way to run these and you are using them at your own risk on an existing cluster.
+
+## Usage
+
+To build/run evertything, use:
+
+```
+make
+```
+
+To target only the e2e tests, use:
+
+```
+make build-controller #created ./bin/terraform-controller
+make e2e
+```
+
+## Running Tests Locally with k3d
+
+If you'd like to run against a local k3s, it is recommended that you use [k3d](https://github.com/rancher/k3d).
+
+To boot a cluster, run:
+
+```
+k3d create --name e2e
+```
+
+Then use the provided config to boot the controller. You can use a command like this:
+
+```
+KUBECONFIG="$(k3d get-kubeconfig --name='e2e')" ./terraform-controller --threads 1
+```
+
+After the controller is running locally, you can run the tests in the same manner:
+
+```
+KUBECONFIG="$(k3d get-kubeconfig --name='e2e')" go test -json -count=1 ./e2e/...
+```
+
+The `count=1` option is needed because tests will be cached if the code doesn't change, even though you are running the tests against a new cluster with no data. In this case, you would want to re-run the tests for a new cluster.
+
+# Initializing
+
+There is an initilization process which mimics `kubectl create -f ./manifests`. This process assumes that you are running a new k3s server, and tests could fail if it tries to make something that already exists. Therefore, if you are doing local development of the e2e tests, it is recommended to delete and recreate your k3s cluster before performing another run.
+
+To automate the process of deleting your k3s cluster before you recreate it, you can use a Run/Debug configuration in Goland or use a one-line shell tool.
+
+# Terraform Module
+
+We use a [test module](https://github.com/luthermonson/terraform-controller-test-module) to test executions. It uses the k8s Terraform provider and creates a ConfigMap. The e2e tests validate that they were created.
+
+Please note that the e2e test module uses the default k3s setting to boot the API server on https://10.43.0.1 as seen [here](https://github.com/luthermonson/terraform-controller-test-module/blob/master/main.tf#L5). If you change the `--api-port` setting, you will need to change this line to match.

e2e/e2e.go

@@ -0,0 +1,237 @@
+package e2e
+
+import (
+	"context"
+	"time"
+
+	tfv1 "github.com/rancher/terraform-controller/pkg/apis/terraformcontroller.cattle.io/v1"
+	tf "github.com/rancher/terraform-controller/pkg/generated/clientset/versioned/typed/terraformcontroller.cattle.io/v1"
+	"github.com/rancher/wrangler/pkg/crd"
+	"github.com/rancher/wrangler/pkg/signals"
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+type E2E struct {
+	ctx        context.Context
+	cs         *kubernetes.Clientset
+	cfg        *rest.Config
+	kubeconfig string
+	namespace  string
+	module_url string
+	crds       []crd.CRD
+}
+
+func NewE2E(namespace, kubeconfig, module string, crdsNames []string) *E2E {
+	cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		logrus.Fatalf("Error building kubeconfig: %s", err.Error())
+	}
+
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		logrus.Fatalf("Error building kubeconfig: %s", err.Error())
+	}
+
+	var crds = make([]crd.CRD, len(crdsNames))
+	for k, v := range crdsNames {
+		crds[k] = crd.FromGV(tfv1.SchemeGroupVersion, v)
+	}
+
+	return &E2E{
+		ctx:        signals.SetupSignalHandler(context.Background()),
+		cs:         cs,
+		cfg:        cfg,
+		kubeconfig: kubeconfig,
+		namespace:  namespace,
+		module_url: module,
+		crds:       crds,
+	}
+}
+
+func (e *E2E) createState() error {
+	cs, err := tf.NewForConfig(e.cfg)
+	if err != nil {
+		return err
+	}
+
+	_, err = cs.States(e.namespace).Create(e.getState())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *E2E) createModule() error {
+	cs, err := tf.NewForConfig(e.cfg)
+	if err != nil {
+		return err
+	}
+
+	_, err = cs.Modules(e.namespace).Create(e.getModule())
+	if err != nil {
+		return err
+	}
+
+	err = wait.Poll(time.Second, 15*time.Second, func() (bool, error) {
+		module, err := cs.Modules(e.namespace).Get(e.generateModuleName(), v12.GetOptions{})
+		if err == nil && module.Status.ContentHash != "" {
+			return true, nil
+		}
+
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+
+		logrus.Printf("Waiting for Module to be processed by terraform-controller: %+v\n", err)
+		return false, err
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *E2E) createVariables() error {
+	_, err := e.cs.CoreV1().Secrets(e.namespace).Create(e.getSecret())
+	if err != nil {
+		return err
+	}
+
+	_, err = e.cs.CoreV1().Secrets(e.namespace).Create(e.getSecretEnv())
+	if err != nil {
+		return err
+	}
+
+	_, err = e.cs.CoreV1().ConfigMaps(e.namespace).Create(e.getConfigMap())
+	if err != nil {
+		return err
+	}
+
+	_, err = e.cs.CoreV1().ConfigMaps(e.namespace).Create(e.getConfigMapEnv())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *E2E) getState() *tfv1.State {
+	return &tfv1.State{
+		ObjectMeta: v12.ObjectMeta{
+			Name:      e.generateStateName(),
+			Namespace: e.namespace,
+		},
+		Spec: tfv1.StateSpec{
+			ModuleName:      e.generateModuleName(),
+			AutoConfirm:     true,
+			DestroyOnDelete: true,
+			Image:           "rancher/terraform-controller-executor:v0.0.10-alpha1",
+			Variables: tfv1.Variables{
+				ConfigNames:    []string{e.generateConfigMapName()},
+				EnvConfigName:  []string{e.generateConfigMapEnvName()},
+				SecretNames:    []string{e.generateSecretName()},
+				EnvSecretNames: []string{e.generateSecretEnvName()},
+			},
+		},
+	}
+}
+
+func (e *E2E) generateStateName() string {
+	return e.namespace + "-state"
+}
+
+func (e *E2E) generateModuleName() string {
+	return e.namespace + "-module"
+}
+
+func (e *E2E) generateSecretName() string {
+	return e.namespace + "-secret"
+}
+
+func (e *E2E) generateSecretEnvName() string {
+	return e.namespace + "-secret-env"
+}
+
+func (e *E2E) getSecret() *v1.Secret {
+	return &v1.Secret{
+		ObjectMeta: v12.ObjectMeta{
+			Name:      e.generateSecretName(),
+			Namespace: e.namespace,
+		},
+		Type: "opaque",
+		StringData: map[string]string{
+			"test_secret": e.namespace,
+		},
+	}
+}
+
+func (e *E2E) getSecretEnv() *v1.Secret {
+	return &v1.Secret{
+		ObjectMeta: v12.ObjectMeta{
+			Name:      e.generateSecretEnvName(),
+			Namespace: e.namespace,
+		},
+		Type: "opaque",
+		StringData: map[string]string{
+			"test_secret_env": e.namespace,
+		},
+	}
+}
+
+func (e *E2E) generateConfigMapName() string {
+	return e.namespace + "-config-map"
+}
+
+func (e *E2E) generateConfigMapEnvName() string {
+	return e.namespace + "-config-map-env"
+}
+
+func (e *E2E) getConfigMap() *v1.ConfigMap {
+	return &v1.ConfigMap{
+		ObjectMeta: v12.ObjectMeta{
+			Name:      e.generateConfigMapName(),
+			Namespace: e.namespace,
+		},
+		Data: map[string]string{
+			"test_config_map": e.namespace,
+		},
+	}
+}
+
+func (e *E2E) getConfigMapEnv() *v1.ConfigMap {
+	return &v1.ConfigMap{
+		ObjectMeta: v12.ObjectMeta{
+			Name:      e.generateConfigMapEnvName(),
+			Namespace: e.namespace,
+		},
+		Data: map[string]string{
+			"test_config_map_env": e.namespace,
+		},
+	}
+}
+
+func (e *E2E) getModule() *tfv1.Module {
+	return &tfv1.Module{
+		ObjectMeta: v12.ObjectMeta{
+			Name:      e.generateModuleName(),
+			Namespace: e.namespace,
+		},
+		Spec: tfv1.ModuleSpec{
+			ModuleContent: tfv1.ModuleContent{
+				Git: tfv1.GitLocation{
+					URL: e.module_url,
+				},
+			},
+		},
+	}
+}