feat(kubernetes): Add PipelinePodsTemplate CRD to define worker-specific Pod defaults for Kubernetes Runtime

.gitignore

@@ -7,6 +7,8 @@
 *.dll
 *.so
 *.dylib
+# install makefile required bins in the bin/ dir
+bin
 
 # Test binary, build with `go test -c`
 *.test

Makefile

@@ -23,6 +23,14 @@ endif
 # create a list of linker flags for building the golang application
 LD_FLAGS = -X github.com/go-vela/worker/version.Commit=${GITHUB_SHA} -X github.com/go-vela/worker/version.Date=${BUILD_DATE} -X github.com/go-vela/worker/version.Tag=${GITHUB_TAG}
 
+# Misc settings for the generating the Kubernetes CRD for the Kubernetes Runtime
+K8S_CRD_OUTPUT_PKG=github.com/go-vela/worker/runtime/kubernetes/generated
+K8S_CRD_INPUT_PKG=github.com/go-vela/worker/runtime/kubernetes/apis
+K8S_CRD_CLIENTSET_PKG_NAME=clientset
+K8S_CRD_CLIENTSET_NAME_VERSIONED=versioned
+K8S_CRD_GROUP=vela
+K8S_CRD_VERSION=v1alpha1
+
 # The `clean` target is intended to clean the workspace
 # and prepare the local changes for submission.
 #
@@ -311,3 +319,76 @@ spec-version-update:
 # Usage: `make spec`
 .PHONY: spec
 spec: spec-gen spec-version-update spec-validate
+
+# The `crd-gen` target is intended to create a k8s CRD client
+# for the kubernetes runtime using k8s.io/code-generator
+#
+# Usage: `make crd-gen`
+.PHONY: crd-client-gen
+crd-client-gen: controller-gen client-gen
+	$(eval TMP := $(shell mktemp -d))
+	@echo
+	@echo "### Generating CRD deepcopy funcs using sig.k8s.io/controller-tools"
+	$(CONTROLLER_GEN) \
+		object:headerFile="runtime/kubernetes/codegen/header.go.txt" \
+		paths="${K8S_CRD_INPUT_PKG}/${K8S_CRD_GROUP}/${K8S_CRD_VERSION}"
+	@echo "### Generating CRD clientset using k8s.io/code-generator"
+	@echo "Generating clientset for ${K8S_CRD_GROUP}:${K8S_CRD_VERSION} at ${K8S_CRD_OUTPUT_PKG}/${K8S_CRD_CLIENTSET_PKG_NAME}"
+	$(CLIENT_GEN) \
+		--clientset-name "${K8S_CRD_CLIENTSET_NAME_VERSIONED}" \
+		--input-base "" \
+		--input \
+			"${K8S_CRD_INPUT_PKG}/${K8S_CRD_GROUP}/${K8S_CRD_VERSION}" \
+		--output-package \
+			"${K8S_CRD_OUTPUT_PKG}/${K8S_CRD_CLIENTSET_PKG_NAME}" \
+		--output-base "${TMP}" \
+		--go-header-file runtime/kubernetes/codegen/header.go.txt
+	@echo "### Copying generated files"
+	rm -rf runtime/kubernetes/generated/clientset
+	cp -r ${TMP}/github.com/go-vela/worker/runtime/kubernetes/* runtime/kubernetes/
+	rm -rf $(TMP)
+	@echo "### CRD clientset created successfully"
+
+# The `crd-manifest` target will call crd-gen to create a k8s crd
+# for the kubernetes runtime.
+#
+# Usage: `make crd`
+.PHONY: crd-manifest
+crd-manifest: controller-gen ## Generate CustomResourceDefinition object.
+	@echo
+	@echo "### Generating CRD manifest using sig.k8s.io/controller-tools"
+	@echo "Generating CRD manifest in runtime/kubernetes/generated"
+	$(CONTROLLER_GEN) crd paths="./..." output:crd:artifacts:config=runtime/kubernetes/generated
+	@echo "### CRD manifest created successfully"
+
+# The `crd` target will call crd-client-gen and crd-manifest
+# to create a k8s CRD for the kubernetes runtime.
+#
+# Usage: `make crd`
+.PHONY: crd
+crd: crd-client-gen crd-manifest
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+.PHONY: controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
+
+CLIENT_GEN = $(shell pwd)/bin/client-gen
+.PHONY: client-gen
+client-gen: ## Download client-gen locally if necessary.
+	$(eval K8S_LIB_VERSION := $(shell go mod graph | grep 'github.com/go-vela/worker k8s.io/client-go@' | sed 's/.*@//'))
+	$(call go-get-tool,$(CLIENT_GEN),k8s.io/code-generator/cmd/client-gen@$(K8S_LIB_VERSION))
+
+# go-get-tool will 'go get' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-get-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef

PROJECT

@@ -0,0 +1,21 @@
+---
+# This file is used in generating code for the Kubernetes Runtime.
+# Look in runtime/kubernetes/apis/ and runtime/kubernetes/generated/
+# for the code that is used or generated by the code generators.
+#
+# For more about this file see:
+#   https://book.kubebuilder.io/reference/project-config.html
+domain: go-vela.github.io
+layout:
+- go.kubebuilder.io/v3
+projectName: worker
+repo: github.com/go-vela/worker
+resources:
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: go-vela.github.io
+  kind: PipelinePodsTemplate
+  path: pkg/runtime/kubernetes/apis/vela/v1alpha1
+  version: v1alpha1
+version: "3"

cmd/vela-worker/exec.go

@@ -56,6 +56,8 @@ func (w *Worker) exec(index int) error {
 		ConfigFile:       w.Config.Runtime.ConfigFile,
 		HostVolumes:      w.Config.Runtime.HostVolumes,
 		Namespace:        w.Config.Runtime.Namespace,
+		PodsTemplateName: w.Config.Runtime.PodsTemplateName,
+		PodsTemplateFile: w.Config.Runtime.PodsTemplateFile,
 		PrivilegedImages: w.Config.Runtime.PrivilegedImages,
 	})
 	if err != nil {

cmd/vela-worker/run.go

@@ -107,6 +107,8 @@ func run(c *cli.Context) error {
 				Driver:           c.String("runtime.driver"),
 				ConfigFile:       c.String("runtime.config"),
 				Namespace:        c.String("runtime.namespace"),
+				PodsTemplateName: c.String("runtime.pods-template-name"),
+				PodsTemplateFile: c.Path("runtime.pods-template-file"),
 				HostVolumes:      c.StringSlice("runtime.volumes"),
 				PrivilegedImages: c.StringSlice("runtime.privileged-images"),
 			},

go.mod

@@ -4,7 +4,6 @@ go 1.17
 
 require (
 	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3
 	github.com/docker/distribution v2.8.1+incompatible
 	github.com/docker/docker v20.10.10+incompatible
 	github.com/docker/go-units v0.4.0
@@ -23,6 +22,7 @@ require (
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
 	k8s.io/client-go v0.23.5
+	sigs.k8s.io/yaml v1.2.0
 )
 
 require (
@@ -34,6 +34,7 @@ require (
 	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
 	github.com/alicebob/miniredis/v2 v2.19.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/containerd/containerd v1.4.13 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
@@ -107,5 +108,4 @@ require (
 	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
 )

runtime/docker/opts_test.go

@@ -5,9 +5,10 @@
 package docker
 
 import (
-	"github.com/sirupsen/logrus"
 	"reflect"
 	"testing"
+
+	"github.com/sirupsen/logrus"
 )
 
 func TestDocker_ClientOpt_WithPrivilegedImages(t *testing.T) {

runtime/flags.go

@@ -36,6 +36,18 @@ var Flags = []cli.Flag{
 		Name:     "runtime.namespace",
 		Usage:    "namespace to use for the runtime (only used by kubernetes)",
 	},
+	&cli.StringFlag{
+		EnvVars:  []string{"VELA_RUNTIME_PODS_TEMPLATE_NAME", "RUNTIME_PODS_TEMPLATE_NAME"},
+		FilePath: "/vela/runtime/pods_template_name",
+		Name:     "runtime.pods-template-name",
+		Usage:    "name of the PipelinePodsTemplate to retrieve from the runtime.namespace (only used by kubernetes)",
+	},
+	&cli.PathFlag{
+		EnvVars:  []string{"VELA_RUNTIME_PODS_TEMPLATE_FILE", "RUNTIME_PODS_TEMPLATE_FILE"},
+		FilePath: "/vela/runtime/pods_template_file",
+		Name:     "runtime.pods-template-file",
+		Usage:    "path to local fallback file containing a PipelinePodsTemplate in YAML (only used by kubernetes; only used if runtime.pods-template-name is not defined)",
+	},
 	&cli.StringSliceFlag{
 		EnvVars:  []string{"VELA_RUNTIME_PRIVILEGED_IMAGES", "RUNTIME_PRIVILEGED_IMAGES"},
 		FilePath: "/vela/runtime/privileged_images",

runtime/kubernetes/apis/doc.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2022 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+// Package apis defines the worker-config CRD and related utilities.
+//
+// Usage:
+//
+// 	import "github.com/go-vela/worker/runtime/kubernetes/apis"
+package apis

runtime/kubernetes/apis/vela/register.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2022 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+package vela
+
+const (
+	// GroupName is the group name used in this package.
+	GroupName = "go-vela.github.io"
+)

runtime/kubernetes/apis/vela/v1alpha1/doc.go

@@ -0,0 +1,15 @@
+// Copyright (c) 2022 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+// +kubebuilder:object:generate=true
+// +kubebuilder:validation:Optional
+// +groupName=go-vela.github.io
+// +groupGoName=Vela
+
+// Package v1alpha1 defines version 1alpha1 of the worker-config CRD.
+//
+// Usage:
+//
+// 	import "github.com/go-vela/worker/runtime/kubernetes/apis/v1alpha1"
+package v1alpha1

runtime/kubernetes/apis/vela/v1alpha1/register.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2022 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"github.com/go-vela/worker/runtime/kubernetes/apis/vela"
+)
+
+// SchemeGroupVersion is group version used to register these objects.
+var SchemeGroupVersion = schema.GroupVersion{Group: vela.GroupName, Version: "v1alpha1"}
+
+// Kind takes an unqualified kind and returns a Group qualified GroupKind.
+func Kind(kind string) schema.GroupKind {
+	return SchemeGroupVersion.WithKind(kind).GroupKind()
+}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource.
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	// SchemeBuilder initializes a scheme builder.
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	// AddToScheme is a global function that registers this API group & version to a scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+// addKnownTypes adds the list of known types to Scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&PipelinePodsTemplate{},
+		&PipelinePodsTemplateList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+
+	return nil
+}