From 76fa25bcb3220ba00adff6faa10b199cea9509cc Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 2 Apr 2022 20:25:42 -0500 Subject: [PATCH] document the PipelinePodsTemplate for k8s runtime --- .../worker/kubernetes/_index.md | 238 +++++++++++++++++- .../administration/worker/reference/_index.md | 24 ++ .../worker/reference/runtime.md | 16 +- 3 files changed, 269 insertions(+), 9 deletions(-) diff --git a/content/administration/worker/kubernetes/_index.md b/content/administration/worker/kubernetes/_index.md index e1fc5020f..f42cbfbed 100644 --- a/content/administration/worker/kubernetes/_index.md +++ b/content/administration/worker/kubernetes/_index.md @@ -28,6 +28,240 @@ This section provides an example of installing the worker with Kubernetes. This example only shows a subset of all possible configuration options. -### Step 1: TODO +### Step 1: Create a Worker Secret and ConfigMap -TODO \ No newline at end of file +You will need to store some env vars in a `Secret`, and the rest can go in a `ConfigMap`. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: vela-worker + namespace: default +data: + # these values are base64 encoded + + VELA_SERVER_SECRET: PHNoYXJlZC1zZWNyZXQ+ + # VELA_SERVER_SECRET: + + VELA_QUEUE_ADDR: cmVkaXM6Ly88cGFzc3dvcmQ+QDxob3N0bmFtZT46PHBvcnQ+LzxkYXRhYmFzZT4= + # VELA_QUEUE_ADDR: redis://@:/ +``` + +Do not store any passwords in the `ConfigMap`. The `ConfigMap` is more convenient for everything else. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: vela-worker + namespace: default +data: + # This might be "http://vela-server:8080" if vela-server is also deployed in k8s. + VELA_SERVER_ADDR: https://vela-server.example.com + + VELA_QUEUE_DRIVER: redis + + VELA_EXECUTOR_DRIVER: linux + + VELA_RUNTIME_DRIVER: kubernetes + VELA_RUNTIME_NAMESPACE: default + VELA_RUNTIME_PODS_TEMPLATE_NAME: pipeline-pods-template + + # do not define VELA_WORKER_ADDR here. See "Create a Worker Deployment" below. + + # VELA_RUNTIME_CONFIG is not needed in-cluster. + # We'll use the auto-mounted ServiceAccount Token. +``` + +And then load them in your cluster. + +```shell +$ kubectl apply -f worker-secret.yaml +$ kubectl apply -f worker-configmap.yaml +``` + +{{% alert title="Note:" color="primary" %}} +For a full list of configuration options, please see the [worker reference](/docs/administration/worker/reference/). +{{% /alert %}} + + +### Step 2: Load the Pipeline Pods Template CRD + +Download Vela's "Pipeline Pods Template" Custom Resource Definition (CRD). +Be sure to replace `v0.14.0` with the version you're installing. + +```shell +$ curl https://raw.githubusercontent.com/go-vela/worker/v0.14.0/runtime/kubernetes/generated/go-vela.github.io_pipelinepodstemplates.yaml -o go-vela.github.io_pipelinepodstemplates.yaml +``` + +And then add the CRD to your cluster. + +```shell +$ kubectl apply -f go-vela.github.io_pipelinepodstemplates.yaml +``` + +### Step 3: Create a Pipeline Pods Template + +The Pipeline Pods Template allows you to define `annotations`, `securityContext`, `dnsConfig` +and other settings that the worker should add to every Pipeline's `Pod`. + +```yaml +apiVersion: go-vela.github.io/v1alpha1 +kind: PipelinePodsTemplate +metadata: + name: pipeline-pods-template # this should match VELA_RUNTIME_PODS_TEMPLATE_NAME + namespace: default +spec: + # spec.template is a subset of what is possible to define in a Deployment's "spec.template". + template: + + metadata: + # custom annotations to add to all pipeline pods + annotations: + example.com/owner: devops + #labels: + + #spec: + # dnsConfig: {} + # dnsPolicy: ClusterFirst + + # nodeSelector: {} + # tolerations: [] + # affinity: {} + + # # These gets applied to all containers in the Pipeline Pod. + # container: + # securityContext: + # capabilities: + # drop: ["ALL"] + # add: [] +``` + +And then your `PipelinePodsTemplate` to your cluster. + +```shell +$ kubectl apply -f pipeline-pods-template.yaml +``` + +### Step 5: Create a ServiceAccount, Role, and RoleBinding + +The Worker needs access to Kubernetes APIs. It uses an auto-mounted ServiceAcount token to do this. + +Here is a ServiceAccount: +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vela-worker + namespace: default +``` + +Here's the Role you'll need: +```yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vela-worker + namespace: default +rules: + - apiGroups: [""] + resources: [pods/log] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [pods] + verbs: [create, patch, get, list, update, watch, delete] + - apiGroups: ["go-vela.github.io"] + resources: [pipelinepodstemplate] + verbs: [get, list, watch] +``` + +And use this RoleBinding: +```yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vela-worker + namespace: default +subjects: + - kind: ServiceAccount + name: vela-worker + namespace: default +roleRef: + kind: Role + name: vela-worker + apiGroup: rbac.authorization.k8s.io +``` + +Then apply each of these to your cluster: + +```shell +$ kubectl apply -f worker-serviceaccount.yaml +$ kubectl apply -f worker-role.yaml +$ kubectl apply -f worker-role-binding.yaml +``` + +### Step 5: Create a Worker Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vela-worker + namespace: default + labels: + app.kubernetes.io/name: vela-worker +spec: + replicas: 1 # Increase this to deploy more worker pods. + selector: + matchLabels: + app.kubernetes.io/name: vela-worker + serviceAccount: vela-worker + containers: + - name: worker + image: target/vela-worker:v0.13.0 + ports: + - {name: http, port: 8080, protocol: TCP} + livenessProbe: + httpGet: {path: /health, port: 8080, scheme: HTTP} + env: + - {name: VELA_WORKER_ADDR_SCHEME, value: http} + - {name: VELA_WORKER_ADDR_PORT, value: "8080"} + - name: VELA_WORKER_POD_NAME + valueFrom: + fieldRef: {fieldPath: metadata.name} + - name: VELA_WORKER_POD_IP + valueFrom: + fieldRef: {fieldPath: status.podIP} + # using the pod name does not get a dns entry without a lot of unnecessary effort. + # So, here we use status.podIP instead of metadata.name + - name: VELA_WORKER_ADDR + value: $(VELA_WORKER_ADDR_SCHEME)://$(WORKER_POD_IP):$(VELA_WORKER_ADDR_PORT) + envFrom: + - configMapRef: + name: vela-worker + - secretRef: + name: vela-worker +``` + +And then load it in your cluster. + +```shell +$ kubectl apply -f worker-deployment.yaml +``` + +### Step 6: Verify the Worker Deployment + +Ensure the worker started up successfully and is running as expected by inspecting details with `kubectl`. + +```shell +$ kubectl describe deployment vela-worker +$ kubectl get pods -l app.kubernetes.io/name=vela-worker +``` + +You can also check the worker logs with `stern`. The following command will tail all of the logs +for pods that start with `vela-worker-`: + +```shell +$ stern vela-worker- +``` \ No newline at end of file diff --git a/content/administration/worker/reference/_index.md b/content/administration/worker/reference/_index.md index e21c6cefb..c70d76968 100644 --- a/content/administration/worker/reference/_index.md +++ b/content/administration/worker/reference/_index.md @@ -236,6 +236,30 @@ This variable sets a namespace (for Kubernetes only) to use for runtime workload The variable can be provided as a `string`. +### VELA_RUNTIME_PODS_TEMPLATE_NAME + +This configuration variable is used by the [runtime component](/docs/administration/worker/reference/runtime/) for the worker. + +Examples using this configuration variable are provided in the [kubernetes runtime documentation](/docs/administration/worker/kubernetes/). + +This variable sets a `PipelinePodsTemplate` name (for Kubernetes only) to use for runtime workloads. +The named template must be in the `VELA_RUNTIME_NAMESPACE`. + +The variable can be provided as a `string`. + +### VELA_RUNTIME_PODS_TEMPLATE_FILE + +This configuration variable is used by the [runtime component](/docs/administration/worker/reference/runtime/) for the worker. + +This variable sets the path to a `PipelinePodsTemplate` YAML file (for Kubernetes only) to use for runtime workloads. +This file is only used if `VELA_RUNTIME_PODS_TEMPLATE_NAME` is empty. + +An example file is provided in the [kubernetes runtime documentation](/docs/administration/worker/kubernetes/). + +This is useful for Kubernetes clusters that do not allow loading CRDs. It is also used for testing Vela. + +The variable can be provided as a `string`. + ### VELA_RUNTIME_PRIVILEGED_IMAGES This configuration variable is used by the [runtime component](/docs/administration/worker/reference/runtime/) for the worker. diff --git a/content/administration/worker/reference/runtime.md b/content/administration/worker/reference/runtime.md index 60c85af1e..95977d935 100644 --- a/content/administration/worker/reference/runtime.md +++ b/content/administration/worker/reference/runtime.md @@ -13,13 +13,15 @@ The runtime environment is used by Vela for executing workload resources and man The following options are used to configure the component: -| Name | Description | Required | Default | Environment Variables | -| --------------------------- | ------------------------------------------------------------ | -------- | ------------------------ | --------------------------------------------------------------- | -| `runtime.config` | path to configuration file for the runtime | `false` | `N/A` | `RUNTIME_CONFIG`
`VELA_RUNTIME_CONFIG` | -| `runtime.driver` | type of client to control and operate runtime | `true` | `docker` | `RUNTIME_DRIVER`
`VELA_RUNTIME_DRIVER` | -| `runtime.namespace` | namespace to use for the runtime (only for kubernetes) | `false` | `N/A` | `RUNTIME_NAMESPACE`
`VELA_RUNTIME_NAMESPACE` | -| `runtime.privileged-images` | images allowed to run in privileged mode for the runtime | `false` | `[ target/vela-docker ]` | `RUNTIME_PRIVILEGED_IMAGES`
`VELA_RUNTIME_PRIVILEGED_IMAGES` | -| `runtime.volumes` | path to host volumes to mount into resources for the runtime | `false` | `N/A` | `RUNTIME_VOLUMES`
`VELA_RUNTIME_VOLUMES` | +| Name | Description | Required | Default | Environment Variables | +|------------------------------|-------------------------------------------------------------------------------------------------|----------|--------------------------|-------------------------------------------------------------------| +| `runtime.config` | path to configuration file for the runtime | `false` | `N/A` | `RUNTIME_CONFIG`
`VELA_RUNTIME_CONFIG` | +| `runtime.driver` | type of client to control and operate runtime | `true` | `docker` | `RUNTIME_DRIVER`
`VELA_RUNTIME_DRIVER` | +| `runtime.namespace` | namespace to use for the runtime (only for kubernetes) | `false` | `N/A` | `RUNTIME_NAMESPACE`
`VELA_RUNTIME_NAMESPACE` | +| `runtime.pods-template-name` | name of the PipelinePodsTemplate to retrieve from the `runtime.namespace` (only for kubernetes) | `false` | `N/A` | `RUNTIME_PODS_TEMPLATE_NAME`
`VELA_RUNTIME_PODS_TEMPLATE_NAME` | +| `runtime.pods-template-file` | path to local fallback file containing a PipelinePodsTemplate in YAML (only for kubernetes) | `false` | `N/A` | `RUNTIME_PODS_TEMPLATE_FILE`
`VELA_RUNTIME_PODS_TEMPLATE_FILE` | +| `runtime.privileged-images` | images allowed to run in privileged mode for the runtime | `false` | `[ target/vela-docker ]` | `RUNTIME_PRIVILEGED_IMAGES`
`VELA_RUNTIME_PRIVILEGED_IMAGES` | +| `runtime.volumes` | path to host volumes to mount into resources for the runtime | `false` | `N/A` | `RUNTIME_VOLUMES`
`VELA_RUNTIME_VOLUMES` | {{% alert title="Note:" color="primary" %}} For more information on these configuration options, please see the [worker reference](/docs/administration/worker/reference/).